Add Memoization to action update urls (#80)

* - Add memoized action url updates.
- Add Pipenv to manage dependecies.
- Update tests to use pytest
- Reorganize the tests directory

* Expanding test_action_update to include a test case for the new memoization

* Ran black over files

* Updating the lint-ci workflow to match the new development flow setup

* Updating the lint-ci to use pipenv

* Adding manual trigger to lint-ci

* fixing Test lint

* trigger ci

* Update lint-workflow/README.md

Co-authored-by: mimartin12 <77340197+mimartin12@users.noreply.github.com>

* Update .github/workflows/lint-ci.yml

Co-authored-by: mimartin12 <77340197+mimartin12@users.noreply.github.com>

* trigger ci

Co-authored-by: mimartin12 <77340197+mimartin12@users.noreply.github.com>

.github/workflows/lint-ci.yml

@@ -6,6 +6,7 @@ on:
   push:
     paths:
       - "lint-workflow/**"
+  workflow_dispatch: {}
 
 jobs:
   CI:
@@ -18,12 +19,15 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install dependencies
+        working-directory: lint-workflow
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pyyaml urllib3
+          pip install pipenv
+          pipenv install --dev
+
       - name: Test lint
-        run: pytest
+        working-directory: lint-workflow
-        working-directory: lint-workflow/tests
+        run: pipenv run pytest tests

lint-workflow/Pipfile

@@ -0,0 +1,15 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pyyaml = "*"
+urllib3 = "*"
+
+[dev-packages]
+black = "*"
+pytest = "*"
+
+[requires]
+python_version = "3.9"

lint-workflow/Pipfile.lock

@@ -0,0 +1,205 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "bc7e46794f16595c77443451de92e409bd86605cdc1d788ee138dab0c5bc13d8"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.9"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "pyyaml": {
+            "hashes": [
+                "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf",
+                "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293",
+                "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b",
+                "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57",
+                "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b",
+                "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4",
+                "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07",
+                "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba",
+                "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9",
+                "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287",
+                "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513",
+                "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0",
+                "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782",
+                "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0",
+                "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92",
+                "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f",
+                "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2",
+                "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc",
+                "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1",
+                "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c",
+                "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86",
+                "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4",
+                "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c",
+                "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34",
+                "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b",
+                "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d",
+                "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c",
+                "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb",
+                "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7",
+                "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737",
+                "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3",
+                "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d",
+                "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358",
+                "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53",
+                "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78",
+                "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803",
+                "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a",
+                "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f",
+                "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174",
+                "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"
+            ],
+            "index": "pypi",
+            "version": "==6.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
+                "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
+            ],
+            "index": "pypi",
+            "version": "==1.26.12"
+        }
+    },
+    "develop": {
+        "attrs": {
+            "hashes": [
+                "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6",
+                "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==22.1.0"
+        },
+        "black": {
+            "hashes": [
+                "sha256:14ff67aec0a47c424bc99b71005202045dc09270da44a27848d534600ac64fc7",
+                "sha256:197df8509263b0b8614e1df1756b1dd41be6738eed2ba9e9769f3880c2b9d7b6",
+                "sha256:1e464456d24e23d11fced2bc8c47ef66d471f845c7b7a42f3bd77bf3d1789650",
+                "sha256:2039230db3c6c639bd84efe3292ec7b06e9214a2992cd9beb293d639c6402edb",
+                "sha256:21199526696b8f09c3997e2b4db8d0b108d801a348414264d2eb8eb2532e540d",
+                "sha256:2644b5d63633702bc2c5f3754b1b475378fbbfb481f62319388235d0cd104c2d",
+                "sha256:432247333090c8c5366e69627ccb363bc58514ae3e63f7fc75c54b1ea80fa7de",
+                "sha256:444ebfb4e441254e87bad00c661fe32df9969b2bf224373a448d8aca2132b395",
+                "sha256:5b9b29da4f564ba8787c119f37d174f2b69cdfdf9015b7d8c5c16121ddc054ae",
+                "sha256:5cc42ca67989e9c3cf859e84c2bf014f6633db63d1cbdf8fdb666dcd9e77e3fa",
+                "sha256:5d8f74030e67087b219b032aa33a919fae8806d49c867846bfacde57f43972ef",
+                "sha256:72ef3925f30e12a184889aac03d77d031056860ccae8a1e519f6cbb742736383",
+                "sha256:819dc789f4498ecc91438a7de64427c73b45035e2e3680c92e18795a839ebb66",
+                "sha256:915ace4ff03fdfff953962fa672d44be269deb2eaf88499a0f8805221bc68c87",
+                "sha256:9311e99228ae10023300ecac05be5a296f60d2fd10fff31cf5c1fa4ca4b1988d",
+                "sha256:974308c58d057a651d182208a484ce80a26dac0caef2895836a92dd6ebd725e0",
+                "sha256:b8b49776299fece66bffaafe357d929ca9451450f5466e997a7285ab0fe28e3b",
+                "sha256:c957b2b4ea88587b46cf49d1dc17681c1e672864fd7af32fc1e9664d572b3458",
+                "sha256:e41a86c6c650bcecc6633ee3180d80a025db041a8e2398dcc059b3afa8382cd4",
+                "sha256:f513588da599943e0cde4e32cc9879e825d58720d6557062d1098c5ad80080e1",
+                "sha256:fba8a281e570adafb79f7755ac8721b6cf1bbf691186a287e990c7929c7692ff"
+            ],
+            "index": "pypi",
+            "version": "==22.10.0"
+        },
+        "click": {
+            "hashes": [
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.3"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
+                "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==21.3"
+        },
+        "pathspec": {
+            "hashes": [
+                "sha256:46846318467efc4556ccfd27816e004270a9eeeeb4d062ce5e6fc7a87c573f93",
+                "sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.10.1"
+        },
+        "platformdirs": {
+            "hashes": [
+                "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788",
+                "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.5.2"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+                "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.0"
+        },
+        "py": {
+            "hashes": [
+                "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
+                "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.11.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
+                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
+            ],
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.0.9"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7",
+                "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"
+            ],
+            "index": "pypi",
+            "version": "==7.1.3"
+        },
+        "tomli": {
+            "hashes": [
+                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+            ],
+            "markers": "python_full_version < '3.11.0a7'",
+            "version": "==2.0.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa",
+                "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"
+            ],
+            "markers": "python_version < '3.10'",
+            "version": "==4.4.0"
+        }
+    }
+}

lint-workflow/README.md

@@ -0,0 +1,28 @@
+# lint-workflow
+
+## Development
+### Requirements
+
+- Python 3.9
+- pipenv
+
+### Setup
+
+```
+pipenv install --dev
+pipenv shell
+```
+
+### Testing
+
+```
+pipenv shell
+pytest tests
+```
+
+### Code Reformatting
+
+```
+pipenv shell
+black .
+```

lint-workflow/lint.py

@@ -12,6 +12,9 @@ PROBLEM_LEVELS = {
 }
 
 
+memoized_action_update_urls = {}
+
+
 class Colors:
     """Class containing color codes for printing strings to output."""
 
@@ -97,17 +100,19 @@ def action_repo_exists(action_id):
     if "bitwarden" in path:
         path_list = path.split("/", 2)
         url = f"https://api.github.com/repos/{path_list[0]}/{path_list[1]}"
-        r = get_github_api_response(url, action_id)
+        response = get_github_api_response(url, action_id)
 
     else:
-        r = get_github_api_response(f"https://api.github.com/repos/{path}", action_id)
+        response = get_github_api_response(
+            f"https://api.github.com/repos/{path}", action_id
+        )
 
-    if r is None:
+    if response is None:
         # Handle github api limit exceed by returning that the action exists without actually checking
         # to prevent false errors on linter output. Only show it as an linter error.
         return True
 
-    if r.status == 404:
+    if response.status == 404:
         return False
 
     return True
@@ -144,46 +149,56 @@ def get_action_update(action_id):
 
     path, *hash = action_id.split("@")
 
+    if path in memoized_action_update_urls:
+        return memoized_action_update_urls[path]
+    else:
         if "bitwarden" in path:
             path_list = path.split("/", 2)
             url = f"https://api.github.com/repos/{path_list[0]}/{path_list[1]}/commits?path={path_list[2]}"
-        r = get_github_api_response(url, action_id)
+            response = get_github_api_response(url, action_id)
-        if r is None:
+            if not response:
                 return None
 
-        sha = json.loads(r.data)[0]["sha"]
+            sha = json.loads(response.data)[0]["sha"]
             if sha not in hash:
-            return f"https://github.com/{path_list[0]}/{path_list[1]}/commit/{sha}"
+                update_url = (
+                    f"https://github.com/{path_list[0]}/{path_list[1]}/commit/{sha}"
+                )
+                memoized_action_update_urls[path] = update_url
+                return update_url
         else:
             # Get tag from latest release
-        r = get_github_api_response(
+            response = get_github_api_response(
                 f"https://api.github.com/repos/{path}/releases/latest", action_id
             )
-        if r is None:
+            if not response:
                 return None
 
-        tag_name = json.loads(r.data)["tag_name"]
+            tag_name = json.loads(response.data)["tag_name"]
 
             # Get the URL to the commit for the tag
-        r = get_github_api_response(
+            response = get_github_api_response(
-            f"https://api.github.com/repos/{path}/git/ref/tags/{tag_name}", action_id
+                f"https://api.github.com/repos/{path}/git/ref/tags/{tag_name}",
+                action_id,
             )
-        if r is None:
+            if not response:
                 return None
 
-        if json.loads(r.data)["object"]["type"] == "commit":
+            if json.loads(response.data)["object"]["type"] == "commit":
-            sha = json.loads(r.data)["object"]["sha"]
+                sha = json.loads(response.data)["object"]["sha"]
             else:
-            url = json.loads(r.data)["object"]["url"]
+                url = json.loads(response.data)["object"]["url"]
                 # Follow the URL and get the commit sha for tags
-            r = get_github_api_response(url, action_id)
+                response = get_github_api_response(url, action_id)
-            if r is None:
+                if not response:
                     return None
 
-            sha = json.loads(r.data)["object"]["sha"]
+                sha = json.loads(response.data)["object"]["sha"]
 
             if sha not in hash:
-            return f"https://github.com/{path}/commit/{sha}"
+                update_url = f"https://github.com/{path}/commit/{sha}"
+                memoized_action_update_urls[path] = update_url
+                return update_url
 
 
 def lint(filename):
@@ -408,4 +423,5 @@ def main(input_args=None):
 
 if __name__ == "__main__":
     return_code = main()
+    print(memoized_action_update_urls)
     sys.exit(return_code)

lint-workflow/tests/configs.py

@@ -0,0 +1 @@
+FIXTURES_DIR = "./tests/fixtures"

lint-workflow/tests/test_action_update.py

@@ -1,6 +1,7 @@
-from lint import get_action_update
 import urllib3 as urllib
 
+from lint import get_action_update, memoized_action_update_urls
+
 http = urllib.PoolManager()
 
 
@@ -13,3 +14,5 @@ def test_action_update():
     r = http.request("GET", update_url)
 
     assert r.status == 200
+
+    assert "actions/checkout" in memoized_action_update_urls

lint-workflow/tests/test_lint.py

@@ -1,8 +1,9 @@
 from lint import lint
+from .configs import FIXTURES_DIR
 
 
 def test_lint(capfd):
-    file_path = "test.yml"
+    file_path = f"{FIXTURES_DIR}/test.yml"
     lint_output = lint(file_path)
     out, err = capfd.readouterr()
     assert (

lint-workflow/tests/test_main.py

@@ -1,17 +1,20 @@
 from lint import main
+from .configs import FIXTURES_DIR
 
 # Tests for argparse inputs and outputs using capsys.readouterr()
 
+FIXTURES_DIR = "./tests/fixtures"
+
 
 def test_main_single_file(capsys):
-    main(["test.yml"])
+    main([f"{FIXTURES_DIR}/test.yml"])
     captured = capsys.readouterr()
     result = captured.out
     assert "test.yml" in result
 
 
 def test_main_multiple_files(capsys):
-    main(["test.yml test-alt.yml"])
+    main([f"{FIXTURES_DIR}/test.yml {FIXTURES_DIR}/test-alt.yml"])
     captured = capsys.readouterr()
     result = captured.out
     assert isinstance(result, str)
@@ -20,7 +23,7 @@ def test_main_multiple_files(capsys):
 
 
 def test_main_folder(capsys):
-    main(["./"])
+    main([f"{FIXTURES_DIR}"])
     captured = capsys.readouterr()
     result = captured.out
     assert isinstance(result, str)
@@ -29,7 +32,7 @@ def test_main_folder(capsys):
 
 
 def test_main_folder_and_files(capsys):
-    main(["test.yml ./"])
+    main([f"{FIXTURES_DIR}/test.yml {FIXTURES_DIR}"])
     captured = capsys.readouterr()
     result = captured.out
     print(result)
@@ -41,9 +44,7 @@ def test_main_not_found(capsys):
     captured = capsys.readouterr()
     result = captured.out
     assert isinstance(result, str)
-    assert (
+    assert 'File(s)/Directory: "not-a-real-file.yml" does not exist, exiting.' in result
-        'File(s)/Directory: "not-a-real-file.yml" does not exist, exiting.' in result
-    )
     # Empty string
     main([""])
     captured = capsys.readouterr()

lint-workflow/tests/test_workflow_files.py

@@ -1,20 +1,35 @@
 import os
+
 from lint import workflow_files
+from .configs import FIXTURES_DIR
 
 
 def test_workflow_files():
     assert workflow_files("") == []
     assert workflow_files("not-a-real-file.yml") == []
-    assert workflow_files("test.yml") == ["test.yml"]
+    assert workflow_files(f"{FIXTURES_DIR}/test.yml") == [f"{FIXTURES_DIR}/test.yml"]
     # multiple files
-    assert workflow_files("test.yml test-alt.yml") == sorted(
+    assert workflow_files(
-        ["test.yml", "test-alt.yml"]
+        f"{FIXTURES_DIR}/test.yml {FIXTURES_DIR}/test-alt.yml"
-    )
+    ) == sorted([f"{FIXTURES_DIR}/test.yml", f"{FIXTURES_DIR}/test-alt.yml"])
     # directory
-    assert workflow_files("../tests") == sorted(set(
+    assert workflow_files(FIXTURES_DIR) == sorted(
-        ["../tests/"+file for file in os.listdir("../tests") if file.endswith((".yml", ".yaml"))]
+        set(
-    ))
+            [
+                f"{FIXTURES_DIR}/{file}"
+                for file in os.listdir(FIXTURES_DIR)
+                if file.endswith((".yml", ".yaml"))
+            ]
+        )
+    )
     # directory and files
-    assert workflow_files("test.yml ../tests") == sorted(set(
+    assert workflow_files(f"{FIXTURES_DIR}/test.yml {FIXTURES_DIR}") == sorted(
-        ["test.yml"] + ["../tests/"+file for file in os.listdir("./") if file.endswith((".yml", ".yaml"))]
+        set(
-    ))
+            [f"{FIXTURES_DIR}/test.yml"]
+            + [
+                f"{FIXTURES_DIR}/{file}"
+                for file in os.listdir(FIXTURES_DIR)
+                if file.endswith((".yml", ".yaml"))
+            ]
+        )
+    )