Arduino-Source/SerialPrograms/Scripts/make_ocr_json.py at main · PizzaTimeJoshua/Arduino-Source · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
"""
Generates an OCR json file for dictionary matching

Set-up:
- Create a file named "target_words.txt". Put all your target dictionary words in this file,
with a new line between each phrase. Put this file in the same folder as this script.
- Adjust the start_line and end_line variables, which are hard-coded.
This helps to limit the script's search of the text dump files.
This helps with dealing with cases where the same English word is used by different parts of the game, but in a different
language, different words may be used. e.g. "Close" in English can be "Nah" or "Schließen" in German.
- Put all the text dump files for all languages in the same folder as this script. You can get the text dump files
from data miners such as https://x.com/mattyoukhana_
  - Ensure the names of the text dump files match the file names in the function `get_text_dump_filename_from_language()`

How it works:
- each target word from "target_words.txt" is converted to a slug.
i.e. spaces are converted to dashes and converted to lowercase
- for each target word, "English.txt" is searched to find the location (line number and column) of the word. All the
text dump files for the various languages are formatted similarly. So, the above location will also point us to
the corresponding target word in other languages.
- the json OCR file is then created based on the slugs and the location of each target word for each of the languages
"""

from typing import TextIO


start_line, end_line = 39560, 39665 # for menu option items
# start_line, end_line = 50255, 51178  # for all moves

def find_location_from_target_word(text_dump_filename: str, target_word: str, start: int, end: int) -> tuple[int, int]:
    with open(text_dump_filename, 'r', encoding='utf-8', errors='ignore') as file:
        for row, line in enumerate(file):
            if row < start:
                continue
            if row > end:
                break
            split_line = line.split("\t")
            for column, cell in enumerate(split_line):
                if cell.strip() == target_word:
                    return row, column
        raise Exception("unable to find target word: " + target_word)

def find_target_word_from_location(text_dump_filename: str, location: tuple[int, int]) -> str:
    target_row = location[0]
    target_column = location[1]
    if target_row < 0:
        raise IndexError("target_row index out of range")
    with open(text_dump_filename, 'r', encoding='utf-8', errors='ignore') as file:
        for row, line in enumerate(file):
            if row == target_row:
                split_line = line.split("\t")
                if target_column < 0 or target_column >= len(split_line):
                    raise IndexError("target_column index out of range")
                for column, cell in enumerate(split_line):
                    if column == target_column:
                        return cell.strip()
    raise IndexError("target_row index out of range")


def get_slug_from_target_word(target_word: str) -> str:
    slug = (target_word.replace(" ", "-")
            .replace("'", "")
            .replace("’", "")
            .replace("é","e")
            .casefold())
    return slug

def generate_ocr_json_file(target_words_filename: str, start: int, end: int):
    list_of_slug_location_pair = get_list_of_slug_location_pair(target_words_filename, "English.txt", start, end)
    json_output_filename = "OCR.json"
    with open(json_output_filename, 'a', encoding='utf-8') as file:
        file.truncate(0) # clear file contents first
        file.write("{\n")
        generate_ocr_json_file_one_language(file, "deu", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "eng", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "spa", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "fra", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "ita", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "kor", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "jpn", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "chi_sim", list_of_slug_location_pair)
        file.write(",\n")
        generate_ocr_json_file_one_language(file, "chi_tra", list_of_slug_location_pair)
        file.write("\n}")


# start and end parameter so we only search certain portions of file.
# This helps with dealing with cases where the same English word is used by different parts of the game,
# but in a different language, different words may be used.
# e.g. "Close" in English can be "Nah" or "Schließen" in German.
def get_list_of_slug_location_pair(target_words_filename: str, text_dump_filename: str, start: int, end: int) -> list[tuple[str, tuple[int, int]]]:
    with open(target_words_filename, 'r', encoding='utf-8', errors='ignore') as file:
        list_of_slug_location_pair = []
        for line in file:
            target_word = line.strip()
            slug = get_slug_from_target_word(target_word)
            location = find_location_from_target_word(text_dump_filename, target_word, start, end)
            slug_location_pair = (slug, location)
            list_of_slug_location_pair.append(slug_location_pair)
    print("Done finding list_of_slug_location_pair")
    return list_of_slug_location_pair


def generate_ocr_json_file_one_language(file: TextIO, language: str, list_of_slug_location_pair: list[tuple[str, tuple[int, int]]]):
    text_dump_filename = get_text_dump_filename_from_language(language)
    file.write("\t" + "\"" + language + "\"" + ": " + "{" + "\n")
    for i, slug_location_pair in enumerate(list_of_slug_location_pair):
        slug = slug_location_pair[0]
        location = slug_location_pair[1]
        target_word = find_target_word_from_location(text_dump_filename, location)
        if i != 0:
            file.write(",\n")
        file.write("\t\t" + "\"" + slug + "\"" + ": " + "[ " + "\"" + target_word + "\"" + " ]")
    file.write("\n\t}")
    print("Done language: " + language)

def get_text_dump_filename_from_language(language: str) -> str:
    text_dump_filename = ""
    match language:
        case "deu":
            text_dump_filename = "Deutsch.txt"
        case "eng":
            text_dump_filename = "English.txt"
        case "spa":
            text_dump_filename = "Español (España).txt"
        case "fra":
            text_dump_filename = "Français.txt"
        case "ita":
            text_dump_filename = "Italiano.txt"
        case "kor":
            text_dump_filename = "한국어.txt"
        case "jpn":
            text_dump_filename = "日本語 (カタカナ).txt"
        case "chi_sim":
            text_dump_filename = "简体中文.txt"
        case "chi_tra":
            text_dump_filename = "繁體中文.txt"

    if text_dump_filename == "":
        raise Exception("Language not recognized")
    return text_dump_filename


generate_ocr_json_file("target_words.txt", start_line, end_line)