[WIP] Recipe to rewrite text in entities

rewrite-text/.gitignore

@@ -0,0 +1,4 @@
+*.csv
+
+# Python
+.coverage

rewrite-text/Makefile

@@ -0,0 +1,10 @@
+.PHONY : clean lint setup setup-dev test
+
+lint:
+	./lint
+
+setup:
+	./setup
+
+setup-dev:
+	pipenv install --dev

rewrite-text/README.md

@@ -0,0 +1,75 @@
+# Rewrite Text
+
+Need to rewrite text in your entity descriptions in your Shortcut workspace (like links to external systems)?
+This recipe helps you accomplish that.
+
+## Usage
+
+You'll need two CSVs: one to specify text substitutions, and one to indicate which entities to rewrite.
+
+### CSV 1: Text Substitutions
+
+Put together a CSV file names `rewrites.csv` with two columns:
+
+- `from`
+- `to`
+
+With this information, the script will comb all non-archived entities within your Shortcut Workspace and rewrite every instance of `from` with `to`.
+Note that it will only match `from` surrounded by non-word characters (a la regular expression `\bexample\b`).
+This is to avoid accidental matches.
+
+### CSV 2: Entities to Rewrite
+
+Next put together a CSV `entities.csv` with the entities you want rewritten.
+This should include the following columns (which match what you get from a Shortcut Workspace CSV export), only one of which should have a value per row:
+
+- `id` (if a story)
+- `epic_id`
+- `iteration_id`
+- `milestone_id` or `objective_id`
+
+If you've already labeled stories and epics that you want to rewrite, you can pass that label id to the `entities_with_labels.py` script and it will produce an output CSV `entities_with_labels.csv` that follows the above pattern:
+
+```shell
+pipenv run python entities_with_labels.py <label-id>
+```
+
+### Running the Code
+
+To download this script's dependencies, run:
+
+```shell
+make setup
+```
+
+Then you can run the rewrite:
+
+```shell
+pipenv run python rewrite_text.py --rewrites rewrites.csv --entities entities.csv
+```
+
+Once you're satisfied that the CSV contains labels you want to archive, pass the `--archive` argument with the name of the CSV:
+
+```shell
+pipenv run python unused_labels.py --archive-labels labels-to-archive.csv
+```
+
+The CSV must have a header with at least an `id` column which is treated as the ID of labels to archive.
+
+Pass `--help` for full usage information.
+
+## Development
+
+Set up your development environment:
+
+```shell
+make setup-dev
+```
+
+Run `pipenv shell` to enter a shell with all Python dependencies loaded.
+
+Run the linter:
+
+```shell
+make lint
+```

rewrite-text/entities_with_label.py

@@ -0,0 +1,43 @@
+# Given a label id, writes a CSV to entities_with_label.csv with either an id or epic_id
+# value on each row for the stories and epics respectively that have this label.
+#
+# The output CSV can be passed to the main rewrite_text.py script's --entities argument
+# to rewrite these entities' textual components.
+
+import re
+import sys
+from lib import sc_get, validate_environment
+
+
+def main(argv):
+    if len(argv) != 2:
+        print(f"Error: Incorrect arguments provided to this script: {argv[1:]}")
+        print("Usage: python entities_with_label.py <label-id>", file=sys.stderr)
+        sys.exit(1)
+    validate_environment()
+    label_id = argv[1]
+    if not bool(re.match(r"^[0-9]+$", label_id)):
+        print(
+            f"Error: Argument must be the label's numeric id, instead received a {type(label_id)} : {label_id}"
+        )
+        print("Usage: python entities_with_label.py <label-id>", file=sys.stderr)
+        sys.exit(1)
+
+    stories = sc_get(f"/labels/{label_id}/stories")
+    epics = sc_get(f"/labels/{label_id}/epics")
+    out_file = "entities_with_label.csv"
+    print(f"Writing {len(stories)} story IDs and {len(epics)} epic IDs to {out_file}")
+    with open(out_file, "w") as f:
+        f.write("id,epic_id\n")
+
+    with open(out_file, "a") as f:
+        for story in stories:
+            f.write(f"{story['id']},\n")
+        for epic in epics:
+            f.write(f",{epic['id']}\n")
+
+    print(f"Finished writing {out_file}")
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

rewrite-text/from_external_links_to_shortcut.py

@@ -0,0 +1,48 @@
+# Given a CSV of entities to migrate, this script queries Shortcut for the stories in it
+# and populates a file named rewrites_from_external_links_to_shortcut.csv with from and to
+# values using the external links as the "from" and the story's application URL as the "to".
+import csv
+import sys
+from lib import sc_get, validate_environment
+
+
+def main(argv):
+    if len(argv) != 2:
+        print(f"Error: Incorrect arguments provided to this script: {argv[1:]}")
+        print(
+            "Usage: python from_external_links_to_shortcut.py <entities.csv>",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    validate_environment()
+    entities_file = argv[1]
+    out_file = "rewrites_from_external_links_to_shortcut.csv"
+
+    with open(entities_file, "r") as in_file:
+        csv_reader = csv.DictReader(in_file)
+
+        print(
+            f"Processing stories from {entities_file}, printing a dot per ten stories..."
+        )
+        with open(out_file, "w") as f:
+            f.write("from,to\n")
+
+        with open(out_file, "a") as f:
+            progress = 0
+            for row in csv_reader:
+                progress += 1
+                if progress % 10 == 0:
+                    print(".", end="", flush=True)
+                id = row["id"]  # Stories are the only entities with external_links
+                if bool(id):
+                    story = sc_get(f"/stories/{id}")
+                    to_url = story["app_url"]
+                    external_links = story["external_links"]
+                    if len(external_links) > 0:
+                        for from_url in external_links:
+                            f.write(f"{from_url},{to_url}\n")
+    print(f"\nFinished writing {out_file}")
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

rewrite-text/lib.py

@@ -0,0 +1,129 @@
+"""Helper functions for communicating with the Shortcut API.
+
+Expects the Shortcut token to be set in the SHORTCUT_API_TOKEN
+environment variable.
+
+"""
+
+from datetime import datetime
+import sys
+import os
+import logging
+
+from pyrate_limiter import Duration, InMemoryBucket, Limiter, Rate  # type: ignore
+import requests
+
+# Logging
+logger = logging.getLogger(__name__)
+
+# Rate limiting. See https://developer.shortcut.com/api/rest/v3#Rate-Limiting
+# The Shortcut API limit is 200 per minute; the 200th request within 60 seconds
+# will receive an HTTP 429 response.
+#
+# The rate limiting config below sets an in-memory limit that is just below
+# Shortcut's rate limit to reduce the possibility of being throttled, and sets
+# the amount of time it will wait once it reaches that limit to just
+# over a minute to account for possible computer clock differences.
+max_requests_per_minute = 200
+rate = Rate(max_requests_per_minute - 5, Duration.MINUTE)
+bucket = InMemoryBucket([rate])
+max_limiter_delay_seconds = 70
+limiter = Limiter(
+    bucket, raise_when_fail=True, max_delay=Duration.SECOND * max_limiter_delay_seconds
+)
+
+
+def rate_mapping(*args, **kwargs):
+    return "shortcut-api-request", 1
+
+
+rate_decorator = limiter.as_decorator()
+
+
+def print_rate_limiting_explanation():
+    printerr(
+        f"""[Note] This script adheres to the Shortcut API rate limit of {max_requests_per_minute} requests per minute.
+       It may pause for up to {max_limiter_delay_seconds} seconds during processing to avoid request throttling."""
+    )
+
+
+# API Helpers
+sc_token = os.getenv("SHORTCUT_API_TOKEN")
+api_url_base = "https://api.app.shortcut.com/api/v3"
+headers = {
+    "Shortcut-Token": sc_token,
+    "Accept": "application/json; charset=utf-8",
+    "Content-Type": "application/json",
+    "User-Agent": "shortcut-api-cookbook/0.0.1-alpha1",
+}
+
+
+@rate_decorator(rate_mapping)
+def sc_get(path, params={}):
+    """
+    Make a GET api call.
+
+    Serializes params as url query parameters.
+    """
+    url = api_url_base + path
+    logger.debug("GET url=%s params=%s headers=%s" % (url, params, headers))
+    resp = requests.get(url, headers=headers, params=params)
+    resp.raise_for_status()
+    return resp.json()
+
+
+@rate_decorator(rate_mapping)
+def sc_post(path, data={}):
+    """Make a POST api call.
+
+    Typically used to create an entity. Other types of requests that
+    are either expensive or need consistent parameter serialization
+    may also use a POST request.  Serializes params as JSON in the
+    request body.
+
+    """
+    url = api_url_base + path
+    logger.debug("POST url=%s params=%s headers=%s" % (url, data, headers))
+    resp = requests.post(url, headers=headers, json=data)
+    logger.debug(f"POST response: {resp.status_code} {resp.text}")
+    resp.raise_for_status()
+    return resp.json()
+
+
+@rate_decorator(rate_mapping)
+def sc_put(path, data={}):
+    """
+    Make a PUT api call.
+
+    Typically used to update an entity.
+    Serializes params as JSON in the request body.
+    """
+    url = api_url_base + path
+    logger.debug("PUT url=%s params=%s headers=%s" % (url, data, headers))
+    resp = requests.put(url, headers=headers, json=data)
+    resp.raise_for_status()
+    return resp.json()
+
+
+def printerr(s):
+    print(s, file=sys.stderr)
+
+
+def validate_environment():
+    """
+    Validate environment settings that must be in place to populate and load
+    the default configuration for this script.
+    """
+    problems = []
+    if sc_token is None:
+        problems.append(
+            " - You must define a SHORTCUT_API_TOKEN environment variable with your Shortcut API token."
+        )
+    if problems:
+        msg = "\n".join(problems)
+        printerr(f"Problems:\n{msg}")
+        sys.exit(1)
+
+
+def now_ts():
+    return datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

rewrite-text/lint

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+pipenv run python -m black .