From a6ce43929baba52b0ad6b5ea43cf644eeb15ca84 Mon Sep 17 00:00:00 2001 From: rctauber Date: Wed, 17 Feb 2021 12:27:13 -0800 Subject: [PATCH 1/2] Update tracked sheet when pushing --- cogs/diff.py | 3 ++- cogs/helpers.py | 43 ++++++++++++++++---------------- cogs/push.py | 65 +++++++++++++++++++++++++++++++------------------ cogs/status.py | 6 +++-- 4 files changed, 69 insertions(+), 48 deletions(-) diff --git a/cogs/diff.py b/cogs/diff.py index d3dcc7c..cd5dbea 100644 --- a/cogs/diff.py +++ b/cogs/diff.py @@ -220,7 +220,8 @@ def diff(paths=None, use_screen=True, verbose=False): remote = f"{cogs_dir}/tracked/{path_name}.tsv" local = details["Path"] if os.path.exists(local) and os.path.exists(remote): - sheet_diff = get_diff(local, remote) + # Consider "remote" the old version to diff off of + sheet_diff = get_diff(remote, local) diffs[sheet_title] = sheet_diff if not diffs: diff --git a/cogs/helpers.py b/cogs/helpers.py index 08c8ee2..7d9f5dc 100644 --- a/cogs/helpers.py +++ b/cogs/helpers.py @@ -130,19 +130,18 @@ def get_data_validation(cogs_dir): return sheet_to_dv_rules -def get_diff(local, remote): - """Return the diff between a local and remote sheet as a list of lines (list of cell values) - with daff 'highlighter' formatting. The 'highlight' is appended to the beginning of the line as: +def get_diff(left, right): + """Return the diff between a left (old) and right (new) sheet as a list of lines (list of cell + values) with daff 'highlighter' formatting. The 'highlight' is appended to the beginning of the + line as: - '+++' for added lines - '->' for changed lines - '...' for omitted rows - '---' for removed lines - - '' for unchanged lines - The remote table is the 'old' version and the local table is the 'new' version.""" - local_data = [] - with open(local, "r") as f: - # Local might be CSV or TSV - if local.endswith("csv"): + - '' for unchanged lines""" + left_data = [] + with open(left, "r") as f: + if left.endswith("csv"): reader = csv.reader(f) else: reader = csv.reader(f, delimiter="\t") @@ -152,36 +151,38 @@ def get_diff(local, remote): # No data header = None if header: - local_data.append(header) + left_data.append(header) for row in reader: if len(row) < len(header): add = [""] * (len(header) - len(row)) row.extend(add) - local_data.append(row) + left_data.append(row) - remote_data = [] - with open(remote, "r") as f: - # Remote is always TSV - reader = csv.reader(f, delimiter="\t") + right_data = [] + with open(right, "r") as f: + if right.endswith("csv"): + reader = csv.reader(f) + else: + reader = csv.reader(f, delimiter="\t") try: header = next(reader) except StopIteration: # No data header = None if header: - remote_data.append(header) + right_data.append(header) for row in reader: if len(row) < len(header): add = [""] * (len(header) - len(row)) row.extend(add) - remote_data.append(row) + right_data.append(row) - if not local_data and not remote_data: + if not right_data and not left_data: return [] - local_table = PythonTableView(local_data) - remote_table = PythonTableView(remote_data) - align = Coopy.compareTables(remote_table, local_table).align() + right_table = PythonTableView(right_data) + left_table = PythonTableView(left_data) + align = Coopy.compareTables(left_table, right_table).align() data_diff = [] table_diff = PythonTableView(data_diff) diff --git a/cogs/push.py b/cogs/push.py index b636d3f..415f9d3 100644 --- a/cogs/push.py +++ b/cogs/push.py @@ -40,7 +40,8 @@ def clear_remote_sheets(spreadsheet, renamed_local): def push_data(cogs_dir, spreadsheet, tracked_sheets, remote_sheets): - """Push all tracked sheets to the spreadsheet. Return updated rows for sheet.tsv.""" + """Push all tracked sheets to the spreadsheet. Update sheets in COGS tracked directory. Return + updated rows for sheet.tsv.""" sheet_rows = [] for sheet_title, details in tracked_sheets.items(): sheet_path = details["Path"] @@ -52,13 +53,17 @@ def push_data(cogs_dir, spreadsheet, tracked_sheets, remote_sheets): if not os.path.exists(sheet_path): logging.warning(f"'{sheet_title}' exists remotely but has not been pulled") continue - with open(sheet_path, "r") as f: - reader = csv.reader(f, delimiter=delimiter) - for row in reader: - row_len = len(row) - if row_len > cols: - cols = row_len - rows.append(row) + with open(sheet_path, "r") as fr: + reader = csv.reader(fr, delimiter=delimiter) + tracked_sheet_title = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower()) + with open(f"{cogs_dir}/tracked/{tracked_sheet_title}.tsv", "w") as fw: + writer = csv.writer(fw, delimiter="\t", lineterminator="\n") + for row in reader: + writer.writerow(row) + row_len = len(row) + if row_len > cols: + cols = row_len + rows.append(row) # Set sheet size if len(rows) < 500: @@ -127,28 +132,40 @@ def push_data_validation(spreadsheet, data_validation, tracked_sheets): show_ui = False if condition.endswith("LIST"): show_ui = True - requests.append({"updateCells": { - "range": { - "sheetId": sheet_id, - "startRowIndex": start_row - 1, - "endRowIndex": end_row, - "startColumnIndex": start_col - 1, - "endColumnIndex": end_col, - }, - "rows": [ - {"values": - {"dataValidation": - {"condition": {"type": condition, "values": value_obj}, - "showCustomUi": show_ui}}}], - "fields": "dataValidation", - }}) + requests.append( + { + "updateCells": { + "range": { + "sheetId": sheet_id, + "startRowIndex": start_row - 1, + "endRowIndex": end_row, + "startColumnIndex": start_col - 1, + "endColumnIndex": end_col, + }, + "rows": [ + { + "values": { + "dataValidation": { + "condition": {"type": condition, "values": value_obj}, + "showCustomUi": show_ui, + } + } + } + ], + "fields": "dataValidation", + } + } + ) if not requests: return try: logging.info(f"adding {len(requests)} data validation rules to spreadsheet") spreadsheet.batch_update({"requests": requests}) except gspread.exceptions.APIError as e: - logging.error(f"Unable to add {len(requests)} data validation rules to spreadsheet\n" + e.response.text) + logging.error( + f"Unable to add {len(requests)} data validation rules to spreadsheet\n" + + e.response.text + ) def push_formats(spreadsheet, id_to_format, sheet_formats): diff --git a/cogs/status.py b/cogs/status.py index bbed190..5a96415 100644 --- a/cogs/status.py +++ b/cogs/status.py @@ -108,10 +108,12 @@ def get_changes(cogs_dir, tracked_sheets, renamed): local_mod = os.path.getmtime(local_path) remote_mod = os.path.getmtime(remote_path) if remote_mod > local_mod: - diff = get_diff(remote_path, local_path) + # Remote is newer + diff = get_diff(local_path, remote_path) new_version = "remote" else: - diff = get_diff(local_path, remote_path) + # Local is newer + diff = get_diff(remote_path, local_path) new_version = "local" if len(diff) > 1: From a3aeb8e7e445659398858e1fcdac5e0112895822 Mon Sep 17 00:00:00 2001 From: rctauber Date: Wed, 24 Feb 2021 07:43:28 -0800 Subject: [PATCH 2/2] Create helper function to get cached path --- cogs/diff.py | 17 +++++++++++------ cogs/fetch.py | 5 +++-- cogs/helpers.py | 6 ++++++ cogs/pull.py | 10 +++++----- cogs/push.py | 9 +++++---- cogs/rm.py | 10 ++++++---- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/cogs/diff.py b/cogs/diff.py index cd5dbea..1d3ea82 100644 --- a/cogs/diff.py +++ b/cogs/diff.py @@ -4,7 +4,13 @@ import tabulate from cogs.exceptions import DiffError -from cogs.helpers import get_diff, get_tracked_sheets, set_logging, validate_cogs_project +from cogs.helpers import ( + get_cached_path, + get_diff, + get_tracked_sheets, + set_logging, + validate_cogs_project, +) def close_screen(stdscr): @@ -216,12 +222,11 @@ def diff(paths=None, use_screen=True, verbose=False): diffs = {} for sheet_title, details in sheets.items(): - path_name = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower()) - remote = f"{cogs_dir}/tracked/{path_name}.tsv" + cached = get_cached_path(cogs_dir, sheet_title) local = details["Path"] - if os.path.exists(local) and os.path.exists(remote): - # Consider "remote" the old version to diff off of - sheet_diff = get_diff(remote, local) + if os.path.exists(local) and os.path.exists(cached): + # Consider remote (cached) the old version to diff off of + sheet_diff = get_diff(cached, local) diffs[sheet_title] = sheet_diff if not diffs: diff --git a/cogs/fetch.py b/cogs/fetch.py index 03b2ba7..5c61310 100644 --- a/cogs/fetch.py +++ b/cogs/fetch.py @@ -9,6 +9,7 @@ import gspread_formatting as gf from cogs.helpers import ( + get_cached_path, get_credentials, get_config, get_format_dict, @@ -412,8 +413,8 @@ def fetch(verbose=False): sheet_notes[st] = cell_to_note # Write values to .cogs/tracked/{sheet title}.tsv - sheet_path = re.sub(r"[^A-Za-z0-9]+", "_", st.lower()).strip("_") - with open(f"{cogs_dir}/tracked/{sheet_path}.tsv", "w") as f: + cached_path = get_cached_path(cogs_dir, st) + with open(cached_path, "w") as f: lines = sheet.get_all_values() writer = csv.writer(f, delimiter="\t", lineterminator="\n") writer.writerows(lines) diff --git a/cogs/helpers.py b/cogs/helpers.py index 7d9f5dc..3a40e3d 100644 --- a/cogs/helpers.py +++ b/cogs/helpers.py @@ -26,6 +26,12 @@ credential_keys = [] +def get_cached_path(cogs_dir, sheet_title): + """Return the path to the cached version of a sheet based on its title.""" + filename = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower()) + return f"{cogs_dir}/tracked/{filename}.tsv" + + def get_cached_sheets(cogs_dir): """Return a list of names of cached sheets from .cogs/tracked. These are any sheets that have been downloaded from the remote spreadsheet into the .cogs directory as TSVs. They may or may diff --git a/cogs/pull.py b/cogs/pull.py index 0031711..b98bbb8 100644 --- a/cogs/pull.py +++ b/cogs/pull.py @@ -5,6 +5,7 @@ import shutil from cogs.helpers import ( + get_cached_path, get_cached_sheets, get_renamed_sheets, get_tracked_sheets, @@ -32,15 +33,14 @@ def pull(verbose=False): remove_sheets = [s for s in cached_sheets if s not in tracked_cached] for sheet_title, details in tracked_sheets.items(): - path_name = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower()) - cached_sheet = f"{cogs_dir}/tracked/{path_name}.tsv" + cached_path = get_cached_path(cogs_dir, sheet_title) local_sheet = details["Path"] - if os.path.exists(cached_sheet): + if os.path.exists(cached_path): logging.info(f"Writing '{sheet_title}' to {local_sheet}") if local_sheet.endswith(".csv"): - copy_to_csv(cached_sheet, local_sheet) + copy_to_csv(cached_path, local_sheet) else: - shutil.copyfile(cached_sheet, local_sheet) + shutil.copyfile(cached_path, local_sheet) for sheet_title in remove_sheets: logging.info(f"Removing '{sheet_title}' from cached sheets") os.remove(f"{cogs_dir}/tracked/{sheet_title}.tsv") diff --git a/cogs/push.py b/cogs/push.py index 415f9d3..14adb3f 100644 --- a/cogs/push.py +++ b/cogs/push.py @@ -10,6 +10,7 @@ get_tracked_sheets, set_logging, validate_cogs_project, + get_cached_path, get_config, get_client_from_config, get_renamed_sheets, @@ -55,8 +56,8 @@ def push_data(cogs_dir, spreadsheet, tracked_sheets, remote_sheets): continue with open(sheet_path, "r") as fr: reader = csv.reader(fr, delimiter=delimiter) - tracked_sheet_title = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower()) - with open(f"{cogs_dir}/tracked/{tracked_sheet_title}.tsv", "w") as fw: + tracked_sheet = get_cached_path(cogs_dir, sheet_title) + with open(tracked_sheet, "w") as fw: writer = csv.writer(fw, delimiter="\t", lineterminator="\n") for row in reader: writer.writerow(row) @@ -98,8 +99,8 @@ def push_data(cogs_dir, spreadsheet, tracked_sheets, remote_sheets): sheet.freeze(frozen_row, frozen_col) # Copy this table into COGS data - path_name = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower()) - with open(f"{cogs_dir}/tracked/{path_name}.tsv", "w") as f: + cached_name = get_cached_path(cogs_dir, sheet_title) + with open(cached_name, "w") as f: writer = csv.writer(f, delimiter="\t", lineterminator="\n") writer.writerows(rows) return sheet_rows diff --git a/cogs/rm.py b/cogs/rm.py index 057cb96..4b09b80 100644 --- a/cogs/rm.py +++ b/cogs/rm.py @@ -1,7 +1,9 @@ import csv +import os from cogs.exceptions import RmError from cogs.helpers import ( + get_cached_path, get_tracked_sheets, set_logging, validate_cogs_project, @@ -36,11 +38,11 @@ def rm(paths, verbose=False): "the spreadsheet must have at least one sheet." ) - # Make sure the titles are valid + # Remove the cached copies for sheet_title in sheets_to_remove.keys(): - if "." in sheet_title or "/" in sheet_title: - # We should probably use a proper way to make sure the file name is in .cogs - raise RmError("Invalid title for sheet, cannot contain . or /") + cached_path = get_cached_path(cogs_dir, sheet_title) + if os.path.exists(cached_path): + os.remove(cached_path) # Update sheet.tsv with open(f"{cogs_dir}/sheet.tsv", "w") as f: