【11】UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff —— XGBoost 模型被当文本读的二进制解码陷阱

UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-41-f37efaf3bea2> in <module>
----> 1 explainer = shap.TreeExplainer(bst)

/.../shap/explainers/tree.py in __init__(self, model, data, model_output, feature_perturbation, **deprecated_options)
    121         self.feature_perturbation = feature_perturbation
    122         self.expected_value = None
--> 123         self.model = TreeEnsemble(model, self.data, self.data_missing, model_output)
    124         self.model_output = model_output

/.../shap/explainers/tree.py in __init__(self, xgb_model)
   1326         self.read_arr('i', 29) # reserved
   1327         self.name_obj_len = self.read('Q')
-> 1328         self.name_obj = self.read_str(self.name_obj_len)
   1329         self.name_gbm_len = self.read('Q')
   1330         self.name_gbm = self.read_str(self.name_gbm_len)

/.../shap/explainers/tree.py in read_str(self, size)
   1456         def read_str(self, size):
   1457             val = self.buf[self.pos:self.pos+size].decode('utf-8')
   1458             self.pos += size
   1459             return val

UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 341: invalid start byte

/* Objects/unicodeobject.c —— PyUnicode_DecodeUTF8Stateful */
while (s < end) {
    Py_UCS4 ch;
    int kind = PyUnicode_KIND(unicode);
    unsigned char byte = (unsigned char)*s;

    if (byte < 0x80) {            /* ASCII 快速路径 */
        ch = byte;
        s++;
    }
    else if (byte < 0xC2) {       /* 0x80~0xC1 永远非法 */
        goto utf8_error;
    }
    else if (byte < 0xE0) {       /* 双字节：0xC2~0xDF */
        /* 校验第二个字节必须是 0x80~0xBF */
        ...
    }
    else if (byte < 0xF0) {       /* 三字节：0xE0~0xEF */
        ...
    }
    else if (byte < 0xF5) {       /* 四字节：0xF0~0xF4 */
        ...
    }
    else {                        /* 0xF5~0xFF：直接进错误分支 */
utf8_error:
        /* 生成 UnicodeDecodeError */
    }
}

# ❌ 错误代码：文本模式读二进制文件
with open('model.xgb', 'r') as f:        # 默认 encoding='utf-8'
    data = f.read()

# ✅ 正确代码：二进制模式
with open('model.xgb', 'rb') as f:
    data = f.read()

# ❌ 错误代码：直接打开 Python2 生成的 pickle
with open('old_data.pkl', 'rb') as f:
    data = pickle.load(f)   # UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2

# ✅ 正确代码：指定 latin1 兜底（Python2 str 本质是 bytes）
with open('old_data.pkl', 'rb') as f:
    data = pickle.load(f, encoding='latin1')

# ❌ 错误代码：默认 UTF-8 读未知编码文件
with open('app.log', 'r') as f:
    lines = f.readlines()   # 文件里混了 GBK 字节就崩

# ✅ 正确代码：先探测，再明确指定
with open('app.log', 'rb') as f:
    raw = f.read()
# 用 chardet 或 file 命令探测编码
# file app.log  →  UTF-8 Unicode text / ISO-8859 text 等
with open('app.log', 'r', encoding='utf-8', errors='replace') as f:
    lines = f.readlines()   # 展示场景用 replace 保命

import requests

# ❌ 错误代码：盲目 .text（requests 按 header 猜编码，猜错就崩）
resp = requests.get('https://example.com/api/data', timeout=10)
print(resp.text)   # 服务端返回 gzip/二进制，或 charset 声明错误

# ✅ 正确代码：先用字节，确认后再解码
resp = requests.get('https://example.com/api/data', timeout=10)
if 'application/json' in resp.headers.get('Content-Type', ''):
    data = resp.json()          # requests 内部正确处理编码
else:
    raw = resp.content          # bytes，永不崩
    text = raw.decode('utf-8', errors='replace')

# ❌ 错误代码：假设所有行都是合法 UTF-8
for row in cursor.fetchall():
    print(row['title'])   # 某行是 GBK 编码的历史数据，直接崩

# ✅ 正确代码：读取时兜底，写入前规范化
for row in cursor.fetchall():
    title = row['title']
    if isinstance(title, bytes):
        title = title.decode('utf-8', errors='replace')
    print(title)

# 1. 先确认文件真实编码（不要猜）
file data.bin
# data.bin: XGBoost model, version: 1.0  ← 二进制格式，不是文本！

# 2. 如果是文本，确认具体编码
file app.log
# app.log: ISO-8859 text          ← 不是 UTF-8！

# 3. 查看非法字节的位置和内容
python3 - <<'EOF'
with open('data.bin', 'rb') as f:
    raw = f.read()
print(f'文件总字节数: {len(raw)}')
print(f'0xff 出现次数: {raw.count(b"\xff")}')
print(f'前 32 字节: {raw[:32].hex(" ")}')
# 前 32 字节: 62 69 6e 61 72 79 ...  ← 明显是二进制头，不是文本
EOF

# 4. 判断数据形态：是"字节数据被当文本"还是"编码设错了"？
#    二进制魔数（model/PNG/ZIP/7z）→ 改 rb 模式
#    文本但编码非 UTF-8（GBK/ISO-8859）→ 改 encoding 参数

# 5. 需要转换编码时，用 iconv 批量转（先备份！）
iconv -f GBK -t UTF-8 old.csv > new.csv && mv new.csv old.csv
# 转换失败会报 "illegal input sequence"，说明文件里混了多种编码

# 一个合法的 UTF-8 中文文本
text = '含光智能'
raw = text.encode('utf-8')
print(raw.hex(' '))
# e5 90 ab e5 85 89 e6 99 ba e8 83 bd

# 错误地按 GBK 解码
try:
    print(raw.decode('gbk'))
except UnicodeDecodeError as e:
    print(f'GBK 解码失败: {e}')

# 1. 造一个假的"模型文件"（纯二进制字节流）
with open('/tmp/fake_model.bin', 'wb') as f:
    f.write(b'\x00\x01\x02\xff\xfe\xfd')   # 含 0xff 的二进制

# 2. 文本模式读取 → 复现报错
try:
    with open('/tmp/fake_model.bin', 'r') as f:
        f.read()
except UnicodeDecodeError as e:
    print(f'复现成功: {e}')
# UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 3: invalid start byte

# 3. 二进制模式读取 → 正常
with open('/tmp/fake_model.bin', 'rb') as f:
    print(f'二进制读取正常: {f.read().hex(" ")}')

# 4. 如果你确实需要"读文本"，显式指定 errors 策略
with open('/tmp/fake_model.bin', 'r', errors='replace') as f:
    print(f'replace 策略: {f.read()!r}')   # '\x00\x01\x02\ufffd\ufffd\ufffd'

import xgboost as xgb

bst = xgb.Booster()
bst.load_model('model.xgb')   # 训练时 save_model 保存的

# 报错发生在 shap.TreeExplainer(bst)
# 因为 SHAP 内部要解析 XGBoost 的模型结构，它假设 buffer 是文本

# 第 1 步：确认 model 文件的字节形态
with open('model.xgb', 'rb') as f:
    head = f.read(16)
print(head.hex(' '))
# 78 67 62 6f 6f 73 74 00 ...  ← "xgboost\0"，二进制头，根本不是文本

# 第 2 步：确认 XGBoost 官方读取方式（永远用库 API，别手动 decode）
# 正确：bst = xgb.Booster(); bst.load_model('model.xgb')
# 错误：open('model.xgb', 'r').read()  ← 崩

# 第 3 步：如果必须手动解析，用二进制模式 + 库提供的序列化格式
# 参考 xgboost 源码里的 binary IO，不要自己 .decode('utf-8')

# 某 Windows 工具导出的配置文件是 UTF-16 LE（带 BOM: FF FE）
# ❌ 错误代码：默认 UTF-8 读取
with open('config.ini', 'r') as f:
    content = f.read()   # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff

# ✅ 正确代码：识别 BOM 后按 UTF-16 读
with open('config.ini', 'rb') as f:
    raw = f.read()
if raw.startswith(b'\xff\xfe'):
    content = raw.decode('utf-16-le')   # UTF-16 LE，去掉 BOM
elif raw.startswith(b'\xef\xbb\xbf'):
    content = raw.decode('utf-8-sig')   # UTF-8 with BOM
else:
    content = raw.decode('utf-8')

# 更省心的方式：codecs 自动识别 BOM
import codecs
with codecs.open('config.ini', 'r', encoding='utf-8-sig') as f:
    content = f.read()