import pprint import re IGNORED_LINES = set(range(6180, 6198)) def readstring(f): bytes = b'' while True: byte = f.read(1) if byte == b'\0': break bytes += byte return bytes.decode('latin-1') def parsedat(filename): with open(filename, "rb") as f: offsets = [] while f.tell() == 0 or f.tell() < offsets[0]: offsets.append(int.from_bytes(f.read(4), 'little')) lines = [] for offset in offsets: if offset != 0: f.seek(offset) lines.append(readstring(f)) else: lines.append('') return lines def dumplines(filename, lines1, lines2): with open(filename, "wt") as f: for line1, line2, lineno in zip(lines1, lines2, range(len(lines1))): f.write(f"[{lineno}]\nUK: {line1}\nUS: {line2}\n") def diffscore(line1, line2): diffset = set(re.split(r"\W", line1.lower())) ^ set(re.split(r"\W", line2.lower())) return sum(len(word) for word in diffset) def scorelines(filename, lines1, lines2): scored = [(diffscore(line1, line2), lineno, line1, line2) for line1, line2, lineno in zip(lines1, lines2, range(len(lines1))) if diffscore(line1, line2) > 0] scored.sort(reverse=True, key=lambda tuple: tuple[0]) with open(filename, "wt") as f: for score, lineno, line1, line2 in scored: if lineno not in IGNORED_LINES: f.write(f"[{lineno}] (diff score: {score})\nUK: {line1}\nUS: {line2}\n") dumplines("all-subtitles-uk-us.txt", parsedat("lang.dat.uk"), parsedat("lang.dat.us")) scorelines("diff-subtitles-uk-us.txt", parsedat("lang.dat.uk"), parsedat("lang.dat.us"))