42 lines
1.5 KiB
Python
42 lines
1.5 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Check COS corpus content"""
|
||
|
|
import os
|
||
|
|
os.environ['ZY_OSS_KEY'] = 'AKIDkQuBQhoiS2OYXWebXLwMbdT7cvAScbbU'
|
||
|
|
os.environ['ZY_OSS_SECRET'] = 'nPoZKArgUJBA4nJenjSxJSQBj5FCj3A4'
|
||
|
|
from qcloud_cos import CosConfig, CosS3Client
|
||
|
|
config = CosConfig(Region='ap-guangzhou', SecretId=os.environ['ZY_OSS_KEY'], SecretKey=os.environ['ZY_OSS_SECRET'])
|
||
|
|
client = CosS3Client(config)
|
||
|
|
bucket = 'sy-finetune-corpus-1317346199'
|
||
|
|
|
||
|
|
# Download and check zhuyuan_full_corpus.jsonl
|
||
|
|
resp = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_full_corpus.jsonl')
|
||
|
|
content = resp['Body'].read().decode('utf-8')
|
||
|
|
lines = content.strip().split('\n')
|
||
|
|
print(f'zhuyuan_full_corpus.jsonl: {len(lines)} 行')
|
||
|
|
print(f'总大小: {len(content)/1024:.1f}KB')
|
||
|
|
print()
|
||
|
|
if lines:
|
||
|
|
import json
|
||
|
|
first = json.loads(lines[0])
|
||
|
|
keys = list(first.keys())
|
||
|
|
print(f'字段: {keys}')
|
||
|
|
if 'messages' in first:
|
||
|
|
msgs = first['messages']
|
||
|
|
print(f'messages 数量: {len(msgs)}')
|
||
|
|
for m in msgs[:3]:
|
||
|
|
print(f' role={m["role"]}, content_len={len(m["content"])}')
|
||
|
|
print()
|
||
|
|
print('第一条前300字符:')
|
||
|
|
s = json.dumps(first, ensure_ascii=False, indent=2)
|
||
|
|
print(s[:300])
|
||
|
|
|
||
|
|
print()
|
||
|
|
print('=== zhuyuan_deep_finetune.jsonl ===')
|
||
|
|
resp2 = client.get_object(Bucket=bucket, Key='corpus/zhuyuan_deep_finetune.jsonl')
|
||
|
|
content2 = resp2['Body'].read().decode('utf-8')
|
||
|
|
lines2 = content2.strip().split('\n')
|
||
|
|
print(f'{len(lines2)} 行')
|
||
|
|
if lines2:
|
||
|
|
d2 = json.loads(lines2[0])
|
||
|
|
print(f'字段: {list(d2.keys())}')
|