104 lines
4.1 KiB
Python
104 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Derive exact duplicate occurrence groups from the canonical GPT segment index.
|
|
|
|
The TCS event is the semantic source. This adapter only groups identical
|
|
text_projection_sha256 values and writes deterministic JSONL in first-seen order.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from collections import OrderedDict
|
|
from pathlib import Path
|
|
|
|
|
|
def file_sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("source", type=Path)
|
|
parser.add_argument("output", type=Path)
|
|
parser.add_argument("--expected-source-sha256", required=True)
|
|
args = parser.parse_args()
|
|
|
|
actual_source_sha = file_sha256(args.source)
|
|
if actual_source_sha != args.expected_source_sha256:
|
|
raise SystemExit(
|
|
f"source SHA mismatch: expected {args.expected_source_sha256}, got {actual_source_sha}"
|
|
)
|
|
|
|
groups: OrderedDict[str, list[dict[str, object]]] = OrderedDict()
|
|
segment_count = 0
|
|
with args.source.open("r", encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, 1):
|
|
record = json.loads(line)
|
|
text_sha = record["text_projection_sha256"]
|
|
groups.setdefault(text_sha, []).append(
|
|
{
|
|
"source_line": line_number,
|
|
"segment_id": record["segment_id"],
|
|
"conversation_ordinal": record["conversation_ordinal"],
|
|
"conversation_id": record["conversation_id"],
|
|
"message_id": record["message_id"],
|
|
"occurred_at_utc": record["occurred_at_utc"],
|
|
"content_parts_canonical_sha256": record[
|
|
"content_parts_canonical_sha256"
|
|
],
|
|
}
|
|
)
|
|
segment_count += 1
|
|
|
|
duplicate_group_count = 0
|
|
duplicate_occurrence_count = 0
|
|
redundant_occurrence_count = 0
|
|
maximum_group_multiplicity = 0
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.output.open("w", encoding="utf-8", newline="\n") as handle:
|
|
for text_sha, occurrences in groups.items():
|
|
if len(occurrences) < 2:
|
|
continue
|
|
duplicate_group_count += 1
|
|
duplicate_occurrence_count += len(occurrences)
|
|
redundant_occurrence_count += len(occurrences) - 1
|
|
maximum_group_multiplicity = max(maximum_group_multiplicity, len(occurrences))
|
|
output_record = {
|
|
"schema": "guanghu.gpt-exact-duplicate-group/v1",
|
|
"corpus_id": "CORPUS-BS-GPT-0001",
|
|
"text_projection_sha256": text_sha,
|
|
"representative_segment_id": occurrences[0]["segment_id"],
|
|
"occurrence_count": len(occurrences),
|
|
"occurrence_segment_ids": [item["segment_id"] for item in occurrences],
|
|
"occurrence_conversation_ordinals": [
|
|
item["conversation_ordinal"] for item in occurrences
|
|
],
|
|
"occurrences": occurrences,
|
|
"semantic_effect": "PROVENANCE_EDGES_ONLY_AFTER_FIRST_OCCURRENCE",
|
|
"promotion_state": "NONE",
|
|
}
|
|
handle.write(json.dumps(output_record, ensure_ascii=False, separators=(",", ":")))
|
|
handle.write("\n")
|
|
|
|
stats = {
|
|
"source_sha256": actual_source_sha,
|
|
"segment_count": segment_count,
|
|
"exact_text_hash_group_count": len(groups),
|
|
"exact_duplicate_group_count": duplicate_group_count,
|
|
"duplicate_occurrence_count": duplicate_occurrence_count,
|
|
"redundant_occurrence_count": redundant_occurrence_count,
|
|
"maximum_group_multiplicity": maximum_group_multiplicity,
|
|
"output_sha256": file_sha256(args.output),
|
|
}
|
|
print(json.dumps(stats, ensure_ascii=False, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|