package org.me.swing.test.t6;

import java.io.*;
import java.nio.charset.*;
import java.nio.file.*;
import java.util.*;

public class FileCharsetUtil {

    private static final String[] ENCODINGS = {
            "UTF-8", "GB18030", "GBK", "GB2312", "Big5", "Shift_JIS", "ISO-8859-1"
    };

    /**
     * 自动检测文件编码
     */
    public static String detectCharset(Path path) throws IOException {
        byte[] head = readFileHead(path, 4096); // 读取前 4KB 判断

        // 1. BOM 检测
        String bomEnc = detectBOM(head);
        if (bomEnc != null) {
            return bomEnc;
        }

        // 2. UTF-8 无 BOM 检测
        if (looksLikeUTF8(head)) {
            return "UTF-8";
        }

        // 3. 逐个尝试常见编码
        for (String enc : ENCODINGS) {
            try {
                String text = new String(head, enc);
                if (isReadable(text)) {
                    return enc;
                }
            } catch (Exception ignored) {}
        }

        // 4. 默认返回 UTF-8
        return "UTF-8";
    }

    /**
     * 智能读取整个文件(小文件适用)
     */
    public static String readAll(String path) throws IOException {
        Path filePath = Paths.get(path);
        byte[] bytes = Files.readAllBytes(filePath);
        String encoding = detectCharset(filePath);
        System.out.println("使用编码读取:" + encoding);

        // 去除 BOM
        bytes = stripBOM(bytes, encoding);
        return new String(bytes, encoding);
    }

    /**
     * 智能读取大文件(逐行读取)
     */
    public static void readLargeFile(String path, LineProcessor processor) throws IOException {
        Path filePath = Paths.get(path);
        String encoding = detectCharset(filePath);
        System.out.println("使用编码逐行读取:" + encoding);

        try (BufferedReader reader = Files.newBufferedReader(filePath, Charset.forName(encoding))) {
            String line;
            while ((line = reader.readLine()) != null) {
                processor.process(line);
            }
        }
    }

    /**
     * 读取文件前 N 个字节(防止大文件内存占用)
     */
    private static byte[] readFileHead(Path path, int maxBytes) throws IOException {
        try (InputStream is = Files.newInputStream(path)) {
            byte[] buffer = new byte[maxBytes];
            int len = is.read(buffer);
            return len == -1 ? new byte[0] : Arrays.copyOf(buffer, len);
        }
    }

    /**
     * 检测 BOM 头
     */
    private static String detectBOM(byte[] bytes) {
        if (bytes.length >= 3 &&
                (bytes[0] & 0xFF) == 0xEF &&
                (bytes[1] & 0xFF) == 0xBB &&
                (bytes[2] & 0xFF) == 0xBF)
            return "UTF-8";
        if (bytes.length >= 2 &&
                (bytes[0] & 0xFF) == 0xFF &&
                (bytes[1] & 0xFF) == 0xFE)
            return "UTF-16LE";
        if (bytes.length >= 2 &&
                (bytes[0] & 0xFF) == 0xFE &&
                (bytes[1] & 0xFF) == 0xFF)
            return "UTF-16BE";
        return null;
    }

    /**
     * 去除 BOM 字节
     */
    private static byte[] stripBOM(byte[] bytes, String encoding) {
        if ("UTF-8".equalsIgnoreCase(encoding) && bytes.length >= 3 &&
                (bytes[0] & 0xFF) == 0xEF &&
                (bytes[1] & 0xFF) == 0xBB &&
                (bytes[2] & 0xFF) == 0xBF)
            return Arrays.copyOfRange(bytes, 3, bytes.length);
        if ("UTF-16LE".equalsIgnoreCase(encoding) && bytes.length >= 2 &&
                (bytes[0] & 0xFF) == 0xFF &&
                (bytes[1] & 0xFF) == 0xFE)
            return Arrays.copyOfRange(bytes, 2, bytes.length);
        if ("UTF-16BE".equalsIgnoreCase(encoding) && bytes.length >= 2 &&
                (bytes[0] & 0xFF) == 0xFE &&
                (bytes[1] & 0xFF) == 0xFF)
            return Arrays.copyOfRange(bytes, 2, bytes.length);
        return bytes;
    }

    /**
     * UTF-8 无 BOM 检测
     */
    private static boolean looksLikeUTF8(byte[] bytes) {
        int i = 0;
        while (i < bytes.length) {
            int b = bytes[i] & 0xFF;
            if (b < 0x80) { // 单字节
                i++;
            } else if ((b >> 5) == 0x6) {
                if (i + 1 >= bytes.length || (bytes[i + 1] & 0xC0) != 0x80)
                    return false;
                i += 2;
            } else if ((b >> 4) == 0xE) {
                if (i + 2 >= bytes.length ||
                        (bytes[i + 1] & 0xC0) != 0x80 ||
                        (bytes[i + 2] & 0xC0) != 0x80)
                    return false;
                i += 3;
            } else if ((b >> 3) == 0x1E) {
                if (i + 3 >= bytes.length ||
                        (bytes[i + 1] & 0xC0) != 0x80 ||
                        (bytes[i + 2] & 0xC0) != 0x80 ||
                        (bytes[i + 3] & 0xC0) != 0x80)
                    return false;
                i += 4;
            } else {
                return false;
            }
        }
        return true;
    }

    /**
     * 可读性判断
     */
    private static boolean isReadable(String s) {
        if (s == null || s.isEmpty()) return false;
        int valid = 0;
        for (char c : s.toCharArray()) {
            if (Character.isWhitespace(c) ||
                    (c >= 32 && c <= 126) ||           // 英文
                    (c >= 0x4E00 && c <= 0x9FA5) ||   // 中文
                    (c >= 0x3040 && c <= 0x30FF) ||   // 日文
                    (c >= 0xAC00 && c <= 0xD7AF)) {   // 韩文
                valid++;
            }
        }
        return valid * 1.0 / s.length() > 0.75;
    }

    /**
     * 行处理接口(适用于大文件读取)
     */
    @FunctionalInterface
    public interface LineProcessor {
        void process(String line) throws IOException;
    }

    // 示例用法
    public static void main(String[] args) throws IOException {
        String path = "D:/DownloadMi/【长篇】红楼梦.txt";

//        // 1️⃣ 获取编码
//        String charset = detectCharset(Paths.get(path));
//        System.out.println("检测结果:" + charset);

        // 2️⃣ 小文件直接读取
//        String content = readAll(path);
//        System.out.println("前 500 字:");
//        System.out.println(content.substring(0, Math.min(content.length(), 500)));
//        System.out.println(content);
//
//        // 3️⃣ 大文件逐行处理
//        System.out.println("\n逐行读取示例(打印前 5 行):");
        final int[] count = {0};
        readLargeFile(path, line -> {
//            if (count[0] < 5) {
//                System.out.println(line);
//            }
//            System.out.println(line);
//            count[0]++;
        });
    }
}

Logo

Agent 垂直技术社区,欢迎活跃、内容共建。

更多推荐