Java 读取大文件 各种编码格式文件
·
package org.me.swing.test.t6;
import java.io.*;
import java.nio.charset.*;
import java.nio.file.*;
import java.util.*;
public class FileCharsetUtil {
private static final String[] ENCODINGS = {
"UTF-8", "GB18030", "GBK", "GB2312", "Big5", "Shift_JIS", "ISO-8859-1"
};
/**
* 自动检测文件编码
*/
public static String detectCharset(Path path) throws IOException {
byte[] head = readFileHead(path, 4096); // 读取前 4KB 判断
// 1. BOM 检测
String bomEnc = detectBOM(head);
if (bomEnc != null) {
return bomEnc;
}
// 2. UTF-8 无 BOM 检测
if (looksLikeUTF8(head)) {
return "UTF-8";
}
// 3. 逐个尝试常见编码
for (String enc : ENCODINGS) {
try {
String text = new String(head, enc);
if (isReadable(text)) {
return enc;
}
} catch (Exception ignored) {}
}
// 4. 默认返回 UTF-8
return "UTF-8";
}
/**
* 智能读取整个文件(小文件适用)
*/
public static String readAll(String path) throws IOException {
Path filePath = Paths.get(path);
byte[] bytes = Files.readAllBytes(filePath);
String encoding = detectCharset(filePath);
System.out.println("使用编码读取:" + encoding);
// 去除 BOM
bytes = stripBOM(bytes, encoding);
return new String(bytes, encoding);
}
/**
* 智能读取大文件(逐行读取)
*/
public static void readLargeFile(String path, LineProcessor processor) throws IOException {
Path filePath = Paths.get(path);
String encoding = detectCharset(filePath);
System.out.println("使用编码逐行读取:" + encoding);
try (BufferedReader reader = Files.newBufferedReader(filePath, Charset.forName(encoding))) {
String line;
while ((line = reader.readLine()) != null) {
processor.process(line);
}
}
}
/**
* 读取文件前 N 个字节(防止大文件内存占用)
*/
private static byte[] readFileHead(Path path, int maxBytes) throws IOException {
try (InputStream is = Files.newInputStream(path)) {
byte[] buffer = new byte[maxBytes];
int len = is.read(buffer);
return len == -1 ? new byte[0] : Arrays.copyOf(buffer, len);
}
}
/**
* 检测 BOM 头
*/
private static String detectBOM(byte[] bytes) {
if (bytes.length >= 3 &&
(bytes[0] & 0xFF) == 0xEF &&
(bytes[1] & 0xFF) == 0xBB &&
(bytes[2] & 0xFF) == 0xBF)
return "UTF-8";
if (bytes.length >= 2 &&
(bytes[0] & 0xFF) == 0xFF &&
(bytes[1] & 0xFF) == 0xFE)
return "UTF-16LE";
if (bytes.length >= 2 &&
(bytes[0] & 0xFF) == 0xFE &&
(bytes[1] & 0xFF) == 0xFF)
return "UTF-16BE";
return null;
}
/**
* 去除 BOM 字节
*/
private static byte[] stripBOM(byte[] bytes, String encoding) {
if ("UTF-8".equalsIgnoreCase(encoding) && bytes.length >= 3 &&
(bytes[0] & 0xFF) == 0xEF &&
(bytes[1] & 0xFF) == 0xBB &&
(bytes[2] & 0xFF) == 0xBF)
return Arrays.copyOfRange(bytes, 3, bytes.length);
if ("UTF-16LE".equalsIgnoreCase(encoding) && bytes.length >= 2 &&
(bytes[0] & 0xFF) == 0xFF &&
(bytes[1] & 0xFF) == 0xFE)
return Arrays.copyOfRange(bytes, 2, bytes.length);
if ("UTF-16BE".equalsIgnoreCase(encoding) && bytes.length >= 2 &&
(bytes[0] & 0xFF) == 0xFE &&
(bytes[1] & 0xFF) == 0xFF)
return Arrays.copyOfRange(bytes, 2, bytes.length);
return bytes;
}
/**
* UTF-8 无 BOM 检测
*/
private static boolean looksLikeUTF8(byte[] bytes) {
int i = 0;
while (i < bytes.length) {
int b = bytes[i] & 0xFF;
if (b < 0x80) { // 单字节
i++;
} else if ((b >> 5) == 0x6) {
if (i + 1 >= bytes.length || (bytes[i + 1] & 0xC0) != 0x80)
return false;
i += 2;
} else if ((b >> 4) == 0xE) {
if (i + 2 >= bytes.length ||
(bytes[i + 1] & 0xC0) != 0x80 ||
(bytes[i + 2] & 0xC0) != 0x80)
return false;
i += 3;
} else if ((b >> 3) == 0x1E) {
if (i + 3 >= bytes.length ||
(bytes[i + 1] & 0xC0) != 0x80 ||
(bytes[i + 2] & 0xC0) != 0x80 ||
(bytes[i + 3] & 0xC0) != 0x80)
return false;
i += 4;
} else {
return false;
}
}
return true;
}
/**
* 可读性判断
*/
private static boolean isReadable(String s) {
if (s == null || s.isEmpty()) return false;
int valid = 0;
for (char c : s.toCharArray()) {
if (Character.isWhitespace(c) ||
(c >= 32 && c <= 126) || // 英文
(c >= 0x4E00 && c <= 0x9FA5) || // 中文
(c >= 0x3040 && c <= 0x30FF) || // 日文
(c >= 0xAC00 && c <= 0xD7AF)) { // 韩文
valid++;
}
}
return valid * 1.0 / s.length() > 0.75;
}
/**
* 行处理接口(适用于大文件读取)
*/
@FunctionalInterface
public interface LineProcessor {
void process(String line) throws IOException;
}
// 示例用法
public static void main(String[] args) throws IOException {
String path = "D:/DownloadMi/【长篇】红楼梦.txt";
// // 1️⃣ 获取编码
// String charset = detectCharset(Paths.get(path));
// System.out.println("检测结果:" + charset);
// 2️⃣ 小文件直接读取
// String content = readAll(path);
// System.out.println("前 500 字:");
// System.out.println(content.substring(0, Math.min(content.length(), 500)));
// System.out.println(content);
//
// // 3️⃣ 大文件逐行处理
// System.out.println("\n逐行读取示例(打印前 5 行):");
final int[] count = {0};
readLargeFile(path, line -> {
// if (count[0] < 5) {
// System.out.println(line);
// }
// System.out.println(line);
// count[0]++;
});
}
}
更多推荐


所有评论(0)