From ae054abb9683a5f95c459d48425aaf5bfffb5f0d Mon Sep 17 00:00:00 2001 From: thatgirlginger Date: Tue, 22 Sep 2026 12:47:16 -0500 Subject: [PATCH 1/3] added method for story file cleaning with custom archives --- 04-Rename-Tags.py | 1 - README.md | 9 +++++ example.yml | 10 +++++ requirements.txt | 2 +- shared_python/Args.py | 23 ++++++++++++ shared_python/Common.py | 12 +++++- story_cleanup/keywords.py | 30 +++++++++++++++ story_cleanup/messages.py | 36 ++++++++++++++++++ story_cleanup/outputs.py | 30 +++++++++++++++ story_cleanup/parsing.py | 38 +++++++++++++++++++ xx-Extract-Notes-Summaries-Other.py | 57 +++++++++++++++++++++++++++++ 11 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 story_cleanup/keywords.py create mode 100644 story_cleanup/messages.py create mode 100644 story_cleanup/outputs.py create mode 100644 story_cleanup/parsing.py create mode 100644 xx-Extract-Notes-Summaries-Other.py diff --git a/04-Rename-Tags.py b/04-Rename-Tags.py index 2316ca5..a0f90a0 100755 --- a/04-Rename-Tags.py +++ b/04-Rename-Tags.py @@ -22,7 +22,6 @@ tw_tags = list(csv.DictReader(csvfile)) tag_headers = tags.tag_export_map total = len(tw_tags) - for cur, row in enumerate(tw_tags): tags.update_tag_row(row) print_progress(cur, total, "tags") diff --git a/README.md b/README.md index 92eebe3..536a052 100755 --- a/README.md +++ b/README.md @@ -255,6 +255,15 @@ On it we can either type (and press enter): Given a comma-separated list of story ids specified in the `story_ids_to_remove` parameter, deletes the corresponding rows from the stories table in the final output database. +## Extract notes, summaries, and commercial links from chapters + +Extracts notes, summaries, content warnings, and + +This is mostly for non-eFiction archives where the author's notes and content warnings are present in the main bodies of work instead of separate fields. If you use this with the 'remove_option' in the properties file populated, the script will create a backup of all works. If you hit an error or KeyboardInterrupt, the script prints out the index of the chapters list you were at. + +note that this does NOT load chapters into the stories table + + ## Parameters diff --git a/example.yml b/example.yml index 25e0e67..82758ef 100755 --- a/example.yml +++ b/example.yml @@ -28,3 +28,13 @@ bookmark_ids_to_remove: /path/to/ids to remove.txt # Chapter file location chapters_path: /path/to/chapter/files/stories chapters_file_extensions: txt, html + + + +# options for story cleanup +# leave blank to ignore story files; any value here will remove flagged text from the original +remove_option: +chapters_backup_path: /path/to/backup/chapter/story/output/files +output_csv: /path/to/output/Desired_CSV_name.csv +# Comma-separated list of types to scan for +scan_types: [notes, warnings, summary, commercial] diff --git a/requirements.txt b/requirements.txt index fd12516..476d90d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ charade==1.0.3 Genshi==0.7.7 ipaddr==2.2.0 keyring==24.2.0 -lxml==4.9.3 +lxml==6.1.3 ndg_httpsclient==0.5.1 pyasn1==0.5.0 pytest==7.4.3 diff --git a/shared_python/Args.py b/shared_python/Args.py index b543b99..59e12d1 100755 --- a/shared_python/Args.py +++ b/shared_python/Args.py @@ -269,3 +269,26 @@ def args_for_08(self): ) self._print_args(self.args) return self.args + + def args_for_sn_extraction(self): + if self.args.chapters_path is None: + self.args.chapters_path = input( + "Location of the text files containing the stories:" + ) + if self.args.remove_option is not None: + if self.args.chapters_backup_path is None: + self.args.chapters_backup_path = input( + "Desired location (full path) of the original text backup" + ) + if os.path.exists(self.args.chapters_backup_path): + os.rmdir(self.args.chapters_backup_path) + + if self.args.output_csv is None: + self.args.output_csv = input( + "Output path for the extracted CSV" + ) + if self.args.scan_types is None: + self.args.scan_types = ['notes', 'warnings', 'summary', 'commercial'] + + self._print_args(self.args) + return self.args \ No newline at end of file diff --git a/shared_python/Common.py b/shared_python/Common.py index 1214162..fb0afbd 100755 --- a/shared_python/Common.py +++ b/shared_python/Common.py @@ -1,6 +1,7 @@ # -- coding: utf-8 -- -import sys from importlib import reload +import os +import sys reload(sys) # sys.setdefaultencoding('utf8') #setdefaultencoding is disabled in Python 3. UTF-8 is also default coding. @@ -13,3 +14,12 @@ def print_progress(cur, total, prog_type="stories"): sys.stdout.write("\r{0}/{1} {2}".format(cur, total, prog_type)) sys.stdout.flush() return cur + +def recursive_story_listdir(main_path): + storyfiles = [x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name] + subdirs = [x.name for x in os.scandir(main_path) if x.is_dir()] + for dir in subdirs: + stories = [x for x in os.listdir(os.path.join(main_path, dir)) if "html" in x or "txt" in x] + for s in stories: + storyfiles.append(os.path.join(dir, s)) + return storyfiles diff --git a/story_cleanup/keywords.py b/story_cleanup/keywords.py new file mode 100644 index 0000000..27612eb --- /dev/null +++ b/story_cleanup/keywords.py @@ -0,0 +1,30 @@ +class Keywords: + def __init__(self): + self.notes = ["note", "Note", "notes", "Notes", "A/n", "a/n", "A/N", "disclaimer", "Disclaimer", "DISCLAIMER"] + self.warnings = ["warning", "Warning", "warnings", "Warnings", "cw", "CW", "content warning", "Content Warning", "trigger warning", + "Trigger Warning", "Content warning", "Trigger warning", "tw", "TW"] + self.summary = ["summary", "Summary", "summaries", "Summaries", "prompt", "Prompt"] + self.commercial = ["Patreon", "patreon", "mailto", "Ko-fi", "ko-fi", "Ko-Fi", "gofundme", "GoFundMe"] + # self.ratings + # self.wordcount + # self.pairings + # self.characters + # others defined + def detect(self, type, text): + match type: + case "notes": + keywords = self.notes + case "warnings": + keywords = self.warnings + case "summary": + keywords = self.summary + case "commercial": + keywords = self.commercial + case _: + raise NameError( + "unsupported type: must be 'notes', 'warnings', 'summary', or 'commercial'; check config file" + ) + for k in keywords: + if k in text: + return True + return False diff --git a/story_cleanup/messages.py b/story_cleanup/messages.py new file mode 100644 index 0000000..3bce060 --- /dev/null +++ b/story_cleanup/messages.py @@ -0,0 +1,36 @@ +from colorama import Fore, Style + +class Messaging: + ''' + class that handles interactivity, message colors, etc + ''' + def __init__(self, text): + self.text = text + self.summary_color = Fore.BLUE + self.notes_color = Fore.MAGENTA + self.warning_color = Fore.LIGHTBLUE_EX + self.commercial_color = Fore.RED + self.additional_lines_color = Fore.LIGHTBLACK_EX + self.general = Fore.GREEN + self.reset = Style.RESET_ALL + def check_init_response(self, type): + match type: + case "summary": + print(self.summary_color + f"{type}:\n" + self.reset + self.text) + case "notes": + print(self.notes_color + f"{type}:\n" + self.reset + self.text) + case "warnings": + print(self.warning_color + f"{type}:\n" + self.reset + self.text) + case "commercial": + print(self.commercial_color + f"{type}:\n" + self.reset + self.text) + r = input(self.general + "Enter 'n' if this is not correct, otherwise press any key to continue: \n" + self.reset) + if r.lower() == "n": + return False + return True + def check_additional_lines(self, preview): + print(f"{self.text}\n") + print(self.additional_lines_color + preview + self.reset) + r = input((self.general + "Enter 'n' if you would like to add the next line to the string identified and removed, otherwise press any key to continue: \n" + self.reset)) + if r == "n": + return True + return False \ No newline at end of file diff --git a/story_cleanup/outputs.py b/story_cleanup/outputs.py new file mode 100644 index 0000000..cf5a79a --- /dev/null +++ b/story_cleanup/outputs.py @@ -0,0 +1,30 @@ +import csv +import os + + +class OutputCSV: + def __init__(self, filename: str, columns: list): + self.filename = filename + self.columns = columns + ['story_identifier'] + def init_csv(self): + with open(self.filename, "w", encoding="utf_8_sig", newline="") as fp: + csvf = csv.DictWriter(fp, fieldnames=self.columns) + csvf.writeheader() + def write_data(self, data: dict): + with open(self.filename, "a", encoding="utf_8_sig", newline="") as f: + csvf = csv.DictWriter(f, fieldnames=self.columns) + csvf.writerow(data) + +class OutputStoryFiles: + def __init__(self, output_path): + self.output_path = output_path + def strip_and_rewrite(self, text, options: dict): + text = "".join(text) + for o in options.keys(): + hit_lines = options[o].split("\n") + for h in hit_lines: + text = text.replace(h, "") + return text + def out_to_file(self, filename, strippedtext): + with open(os.path.join(self.output_path, filename), "w") as f: + f.write(strippedtext) diff --git a/story_cleanup/parsing.py b/story_cleanup/parsing.py new file mode 100644 index 0000000..43ca135 --- /dev/null +++ b/story_cleanup/parsing.py @@ -0,0 +1,38 @@ +from .keywords import Keywords +from .messages import Messaging + +class Parser: + def __init__(self, textlines: list, options: list): + self.textlines = textlines + self.max = len(self.textlines) + self.options_dict = {} + for opt in options: + self.options_dict.update({opt:""}) + self.end_message = "***no more lines to preview, press enter to continue***" + self.kwords = Keywords() + + def detection_loop(self, type: str, i: int): + m = Messaging(self.textlines[i]) + if m.check_init_response(type): + local_count = i + 1 + txt = self.textlines[i] + try: + a = m.check_additional_lines(self.textlines[local_count]) + except IndexError: + input(self.end_message) + return txt + while a and local_count < self.max: + preview = self.textlines[local_count + 1] + txt = txt + self.textlines[local_count] + a = m.check_additional_lines(preview) + local_count += 1 + return txt + return "" + + def parse_lines(self): + # this is per story file + for i in range(0, int(self.max)): + for o in self.options_dict.keys(): + if self.kwords.detect(o, self.textlines[i]): + self.options_dict[o] = self.options_dict[o] + "\n" + self.detection_loop(o, i) + return self.options_dict diff --git a/xx-Extract-Notes-Summaries-Other.py b/xx-Extract-Notes-Summaries-Other.py new file mode 100644 index 0000000..7497be0 --- /dev/null +++ b/xx-Extract-Notes-Summaries-Other.py @@ -0,0 +1,57 @@ +import os +import shutil + +from shared_python.Args import Args +from shared_python.Common import recursive_story_listdir + +from story_cleanup.outputs import OutputCSV, OutputStoryFiles +from story_cleanup.parsing import Parser + + +#TODO: improve logging +''' +For archives where authors' notes and summaries are in the main body of text, this script scans for and removes them +note that this does /not/ load fields into the database tables, only extracts them to a CSV (and deletes if necessary) +''' + +if __name__ == "__main__": + args_obj = Args() + args = args_obj.args_for_sn_extraction() + stories = recursive_story_listdir(args.chapters_path) + log = args_obj.logger_with_filename() + + options = args.scan_types + to_remove = args.remove_option + + csv = OutputCSV(args.output_csv, options) + j = int(input("the index you left off at, if you are starting from scratch enter 0\n")) + + if to_remove is not None and j == 0: + shutil.copytree(args.chapters_path, args.chapters_backup_path) + rewrite = OutputStoryFiles(args.chapters_path) + + try: + # TODO: function that queries db for chapter and filenames + if j == 0: + csv.init_csv() + for s in stories[j:]: + log.info(f"story: {s}") + fname = os.path.join(args.chapters_path, s) + with open(fname, "r") as f: + text = f.readlines() + parser = Parser(text, options) + hits = parser.parse_lines() + + if to_remove is not None: + new_text = rewrite.strip_and_rewrite(text, hits) + rewrite.out_to_file(s, new_text) + + hits.update({'story_identifier':s}) + csv.write_data(hits) + j += 1 + + except KeyboardInterrupt: + print(f"progress at restart: {j}") + except Exception as e: + print(f"there's been an error: {e}\n") + print(f"progress at restart: {j}") From 62affd1cb70f42506f96161dfb90f5cc9ee8d2c8 Mon Sep 17 00:00:00 2001 From: thatgirlginger Date: Tue, 22 Sep 2026 12:50:39 -0500 Subject: [PATCH 2/3] updated mistake in README addition --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 536a052..9c54361 100755 --- a/README.md +++ b/README.md @@ -257,7 +257,7 @@ rows from the stories table in the final output database. ## Extract notes, summaries, and commercial links from chapters -Extracts notes, summaries, content warnings, and +Extracts notes, summaries, content warnings, and commercial links from story files This is mostly for non-eFiction archives where the author's notes and content warnings are present in the main bodies of work instead of separate fields. If you use this with the 'remove_option' in the properties file populated, the script will create a backup of all works. If you hit an error or KeyboardInterrupt, the script prints out the index of the chapters list you were at. From bca989a2668c01973bb675c96847676ee8351ec2 Mon Sep 17 00:00:00 2001 From: thatgirlginger Date: Wed, 23 Sep 2026 10:26:32 -0500 Subject: [PATCH 3/3] ruff reformatting --- shared_python/Args.py | 10 +++--- shared_python/Common.py | 11 ++++-- story_cleanup/keywords.py | 52 ++++++++++++++++++++++++++--- story_cleanup/messages.py | 24 ++++++++++--- story_cleanup/outputs.py | 7 +++- story_cleanup/parsing.py | 7 ++-- xx-Extract-Notes-Summaries-Other.py | 16 +++++---- 7 files changed, 99 insertions(+), 28 deletions(-) diff --git a/shared_python/Args.py b/shared_python/Args.py index 59e12d1..4a7c1ba 100755 --- a/shared_python/Args.py +++ b/shared_python/Args.py @@ -282,13 +282,11 @@ def args_for_sn_extraction(self): ) if os.path.exists(self.args.chapters_backup_path): os.rmdir(self.args.chapters_backup_path) - + if self.args.output_csv is None: - self.args.output_csv = input( - "Output path for the extracted CSV" - ) + self.args.output_csv = input("Output path for the extracted CSV") if self.args.scan_types is None: - self.args.scan_types = ['notes', 'warnings', 'summary', 'commercial'] + self.args.scan_types = ["notes", "warnings", "summary", "commercial"] self._print_args(self.args) - return self.args \ No newline at end of file + return self.args diff --git a/shared_python/Common.py b/shared_python/Common.py index fb0afbd..c8b2509 100755 --- a/shared_python/Common.py +++ b/shared_python/Common.py @@ -15,11 +15,18 @@ def print_progress(cur, total, prog_type="stories"): sys.stdout.flush() return cur + def recursive_story_listdir(main_path): - storyfiles = [x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name] + storyfiles = [ + x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name + ] subdirs = [x.name for x in os.scandir(main_path) if x.is_dir()] for dir in subdirs: - stories = [x for x in os.listdir(os.path.join(main_path, dir)) if "html" in x or "txt" in x] + stories = [ + x + for x in os.listdir(os.path.join(main_path, dir)) + if "html" in x or "txt" in x + ] for s in stories: storyfiles.append(os.path.join(dir, s)) return storyfiles diff --git a/story_cleanup/keywords.py b/story_cleanup/keywords.py index 27612eb..b274589 100644 --- a/story_cleanup/keywords.py +++ b/story_cleanup/keywords.py @@ -1,15 +1,57 @@ class Keywords: def __init__(self): - self.notes = ["note", "Note", "notes", "Notes", "A/n", "a/n", "A/N", "disclaimer", "Disclaimer", "DISCLAIMER"] - self.warnings = ["warning", "Warning", "warnings", "Warnings", "cw", "CW", "content warning", "Content Warning", "trigger warning", - "Trigger Warning", "Content warning", "Trigger warning", "tw", "TW"] - self.summary = ["summary", "Summary", "summaries", "Summaries", "prompt", "Prompt"] - self.commercial = ["Patreon", "patreon", "mailto", "Ko-fi", "ko-fi", "Ko-Fi", "gofundme", "GoFundMe"] + self.notes = [ + "note", + "Note", + "notes", + "Notes", + "A/n", + "a/n", + "A/N", + "disclaimer", + "Disclaimer", + "DISCLAIMER", + ] + self.warnings = [ + "warning", + "Warning", + "warnings", + "Warnings", + "cw", + "CW", + "content warning", + "Content Warning", + "trigger warning", + "Trigger Warning", + "Content warning", + "Trigger warning", + "tw", + "TW", + ] + self.summary = [ + "summary", + "Summary", + "summaries", + "Summaries", + "prompt", + "Prompt", + ] + self.commercial = [ + "Patreon", + "patreon", + "mailto", + "Ko-fi", + "ko-fi", + "Ko-Fi", + "gofundme", + "GoFundMe", + ] # self.ratings # self.wordcount # self.pairings # self.characters # others defined + def detect(self, type, text): match type: case "notes": diff --git a/story_cleanup/messages.py b/story_cleanup/messages.py index 3bce060..bdf7e44 100644 --- a/story_cleanup/messages.py +++ b/story_cleanup/messages.py @@ -1,9 +1,11 @@ from colorama import Fore, Style + class Messaging: - ''' + """ class that handles interactivity, message colors, etc - ''' + """ + def __init__(self, text): self.text = text self.summary_color = Fore.BLUE @@ -13,6 +15,7 @@ def __init__(self, text): self.additional_lines_color = Fore.LIGHTBLACK_EX self.general = Fore.GREEN self.reset = Style.RESET_ALL + def check_init_response(self, type): match type: case "summary": @@ -23,14 +26,25 @@ def check_init_response(self, type): print(self.warning_color + f"{type}:\n" + self.reset + self.text) case "commercial": print(self.commercial_color + f"{type}:\n" + self.reset + self.text) - r = input(self.general + "Enter 'n' if this is not correct, otherwise press any key to continue: \n" + self.reset) + r = input( + self.general + + "Enter 'n' if this is not correct, otherwise press any key to continue: \n" + + self.reset + ) if r.lower() == "n": return False return True + def check_additional_lines(self, preview): print(f"{self.text}\n") print(self.additional_lines_color + preview + self.reset) - r = input((self.general + "Enter 'n' if you would like to add the next line to the string identified and removed, otherwise press any key to continue: \n" + self.reset)) + r = input( + ( + self.general + + "Enter 'n' if you would like to add the next line to the string identified and removed, otherwise press any key to continue: \n" + + self.reset + ) + ) if r == "n": return True - return False \ No newline at end of file + return False diff --git a/story_cleanup/outputs.py b/story_cleanup/outputs.py index cf5a79a..42e75d8 100644 --- a/story_cleanup/outputs.py +++ b/story_cleanup/outputs.py @@ -5,19 +5,23 @@ class OutputCSV: def __init__(self, filename: str, columns: list): self.filename = filename - self.columns = columns + ['story_identifier'] + self.columns = columns + ["story_identifier"] + def init_csv(self): with open(self.filename, "w", encoding="utf_8_sig", newline="") as fp: csvf = csv.DictWriter(fp, fieldnames=self.columns) csvf.writeheader() + def write_data(self, data: dict): with open(self.filename, "a", encoding="utf_8_sig", newline="") as f: csvf = csv.DictWriter(f, fieldnames=self.columns) csvf.writerow(data) + class OutputStoryFiles: def __init__(self, output_path): self.output_path = output_path + def strip_and_rewrite(self, text, options: dict): text = "".join(text) for o in options.keys(): @@ -25,6 +29,7 @@ def strip_and_rewrite(self, text, options: dict): for h in hit_lines: text = text.replace(h, "") return text + def out_to_file(self, filename, strippedtext): with open(os.path.join(self.output_path, filename), "w") as f: f.write(strippedtext) diff --git a/story_cleanup/parsing.py b/story_cleanup/parsing.py index 43ca135..56544c7 100644 --- a/story_cleanup/parsing.py +++ b/story_cleanup/parsing.py @@ -1,13 +1,14 @@ from .keywords import Keywords from .messages import Messaging + class Parser: def __init__(self, textlines: list, options: list): self.textlines = textlines self.max = len(self.textlines) self.options_dict = {} for opt in options: - self.options_dict.update({opt:""}) + self.options_dict.update({opt: ""}) self.end_message = "***no more lines to preview, press enter to continue***" self.kwords = Keywords() @@ -34,5 +35,7 @@ def parse_lines(self): for i in range(0, int(self.max)): for o in self.options_dict.keys(): if self.kwords.detect(o, self.textlines[i]): - self.options_dict[o] = self.options_dict[o] + "\n" + self.detection_loop(o, i) + self.options_dict[o] = ( + self.options_dict[o] + "\n" + self.detection_loop(o, i) + ) return self.options_dict diff --git a/xx-Extract-Notes-Summaries-Other.py b/xx-Extract-Notes-Summaries-Other.py index 7497be0..171fb9b 100644 --- a/xx-Extract-Notes-Summaries-Other.py +++ b/xx-Extract-Notes-Summaries-Other.py @@ -8,11 +8,11 @@ from story_cleanup.parsing import Parser -#TODO: improve logging -''' +# TODO: improve logging +""" For archives where authors' notes and summaries are in the main body of text, this script scans for and removes them note that this does /not/ load fields into the database tables, only extracts them to a CSV (and deletes if necessary) -''' +""" if __name__ == "__main__": args_obj = Args() @@ -24,12 +24,14 @@ to_remove = args.remove_option csv = OutputCSV(args.output_csv, options) - j = int(input("the index you left off at, if you are starting from scratch enter 0\n")) + j = int( + input("the index you left off at, if you are starting from scratch enter 0\n") + ) if to_remove is not None and j == 0: shutil.copytree(args.chapters_path, args.chapters_backup_path) rewrite = OutputStoryFiles(args.chapters_path) - + try: # TODO: function that queries db for chapter and filenames if j == 0: @@ -45,8 +47,8 @@ if to_remove is not None: new_text = rewrite.strip_and_rewrite(text, hits) rewrite.out_to_file(s, new_text) - - hits.update({'story_identifier':s}) + + hits.update({"story_identifier": s}) csv.write_data(hits) j += 1