From dc7511ca9a011f62e27a2d9ea76d90f4a8e12dd0 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 07:37:07 -0500 Subject: [PATCH 01/10] run pre-commit --- .github/workflows/deploy-pypi-list.yml | 2 +- requirements.txt | 1 - setup.py | 3 ++- site/index.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-pypi-list.yml b/.github/workflows/deploy-pypi-list.yml index 2e01045..fbff830 100644 --- a/.github/workflows/deploy-pypi-list.yml +++ b/.github/workflows/deploy-pypi-list.yml @@ -16,7 +16,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - + - name: Setup Python uses: actions/setup-python@v2.2.2 with: diff --git a/requirements.txt b/requirements.txt index 9f4827c..e9a3f53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ kedro requests - diff --git a/setup.py b/setup.py index 5267400..2f7162b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ -from setuptools import setup, find_packages from pathlib import Path +from setuptools import find_packages, setup + requires = Path("requirements.txt").read_text().split() dev_requires = Path("requirements_dev.txt").read_text().split() diff --git a/site/index.html b/site/index.html index c26cf40..f7a0e27 100644 --- a/site/index.html +++ b/site/index.html @@ -30,6 +30,6 @@

pypi-list

Checkout the available pypi packages that are single words in available.json - + From 65241d7751603a96c7a0ef432c3011a1169223b0 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 07:47:54 -0500 Subject: [PATCH 02/10] Add interrogate to pre-commit-config --- .pre-commit-config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e58ab5..f9cf0a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,3 +39,11 @@ repos: hooks: - id: mypy exclude: tests/ + - repo: https://github.com/econchick/interrogate + rev: '1.2.0' + hooks: + - id: interrogate + description: check for docstring coverage + entry: interrogate -f 100 -iIvv + types: [python] + exclude: docs From f7eb84c03b2d827a7d29db56412bc1b6b0c2a90d Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 07:51:35 -0500 Subject: [PATCH 03/10] add instructions for using setup.py --- setup.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup.py b/setup.py index 2f7162b..2ed52ba 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,18 @@ +""" +pypi-list uses setup tools for packaging. + +To Build pypi-list as a Python package + + $ python setup.py sdist bdist_wheel + +Regular install + + $ pip install -e . + +To setup local Development + + $ pip install -e ".[dev]" +""" from pathlib import Path from setuptools import find_packages, setup From 6ecefb482f91b1bc22136ee39cd670fb2eddb1d5 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 08:11:13 -0500 Subject: [PATCH 04/10] add docstrings to project --- pypi_list/__init__.py | 44 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/pypi_list/__init__.py b/pypi_list/__init__.py index b25cb95..e442e96 100644 --- a/pypi_list/__init__.py +++ b/pypi_list/__init__.py @@ -4,7 +4,7 @@ pypi-list Listing python packages from pypi, and finding available single word packages. -## Run the Pipeline +## Run the Pipeline at the command line ``` bash # Run with existing package data @@ -13,6 +13,16 @@ python pypi-list.py # Full run python pypi-list.py --full ``` + +## Run the Pipeline with a python repl + +``` python +from pypi_list import run_project + +run_project() # run local datasets only +run_project(full=True) # run full pipeline including network requests +``` + """ import logging @@ -29,6 +39,7 @@ __version__ = "0.2.0" def get_body(packages): + """Get the body tag from the full page html.""" tag = "\n" index = packages.find(tag) + len(tag) @@ -135,11 +146,36 @@ catalog = DataCatalog( runner = SequentialRunner() -def main(): +def run_project(full=None): + """ + Run the project. + + Parameters + -------- + full : bool + runs the full pipeline if True + skips network calls if False + checks sys.arv for --full if None + + Returns + -------- + None + + Examples + -------- + >>> from pypi_list import run_project + >>> run_project() # run local datasets only + >>> run_project(full=True) # run full pipeline including network requests + + """ import sys - if "--full" in sys.argv: + if "--full" in sys.argv and full is None: + full = True + + if full: runner.run(pipeline, catalog) + else: runner.run( Pipeline([node for node in pipeline.nodes if "raw" not in node.name]), @@ -148,4 +184,4 @@ def main(): if __name__ == "__main__": - main() + run_project() From 39f672cb2167487045951eef8aee991c579cff2a Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 08:17:29 -0500 Subject: [PATCH 05/10] add type hinting --- pypi_list/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pypi_list/__init__.py b/pypi_list/__init__.py index e442e96..a46a5ee 100644 --- a/pypi_list/__init__.py +++ b/pypi_list/__init__.py @@ -25,6 +25,7 @@ run_project(full=True) # run full pipeline including network requests """ import logging +from typing import List, Optional import requests from kedro.extras.datasets.json import JSONDataSet @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) __version__ = "0.2.0" -def get_body(packages): +def get_body(packages: str) -> str: """Get the body tag from the full page html.""" tag = "\n" @@ -48,7 +49,7 @@ def get_body(packages): return packages -def remove_html_tags(text): +def remove_html_tags(text: str) -> List[str]: """Remove html tags from a string""" import re @@ -146,7 +147,7 @@ catalog = DataCatalog( runner = SequentialRunner() -def run_project(full=None): +def run_project(full: Optional[bool] = None): """ Run the project. From e58af1a09295eb9ba65296cbb2b8a2d38f98e9e4 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 08:22:59 -0500 Subject: [PATCH 06/10] Add pre-commit intstructions --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 74df945..579edc9 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,11 @@ This pipeline run daily at `0:0` to generate a `packages.json`, ``` bash pip install -e ".[dev]" ``` + +This project includes a `pre-commit-config`, this ensures that each commit made +to the project passes all checks that are also ran in CI. To setup pre-commit +in your development environment run the follow command. + +``` bash +pre-commit install +``` From 89d29fbc5ddea72aed99f01f4609788fb9c9f4b3 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 08:25:43 -0500 Subject: [PATCH 07/10] FIX correct version in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4ef4557..b192d83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.kedro] package_name = "pypi_list" project_name = "Pypi List" -project_version = "0.0.2" +project_version = "0.2.0" [tool.isort] multi_line_output = 3 From 355034216c5a882bdcde4830e51cfbc7beaa4763 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 08:26:39 -0500 Subject: [PATCH 08/10] =?UTF-8?q?Bump=20version:=200.2.0=20=E2=86=92=200.3?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pypi_list/__init__.py | 2 +- pyproject.toml | 2 +- setup.cfg | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pypi_list/__init__.py b/pypi_list/__init__.py index a46a5ee..66b1340 100644 --- a/pypi_list/__init__.py +++ b/pypi_list/__init__.py @@ -36,7 +36,7 @@ from kedro.runner.sequential_runner import SequentialRunner logger = logging.getLogger(__name__) -__version__ = "0.2.0" +__version__ = "0.3.0" def get_body(packages: str) -> str: diff --git a/pyproject.toml b/pyproject.toml index b192d83..1720380 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.kedro] package_name = "pypi_list" project_name = "Pypi List" -project_version = "0.2.0" +project_version = "0.3.0" [tool.isort] multi_line_output = 3 diff --git a/setup.cfg b/setup.cfg index 1db8deb..4685200 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.0 +current_version = 0.3.0 commit = True tag = True diff --git a/setup.py b/setup.py index 2ed52ba..08272ee 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ dev_requires = Path("requirements_dev.txt").read_text().split() setup( name="pypi_list", - version="0.2.0", + version="0.3.0", packages=find_packages(), install_requires=requires, extras_require={ From 9e2abc427b7ba69d4d0b9e492b8fe5fb60bb04b7 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 08:31:25 -0500 Subject: [PATCH 09/10] Fix refactored main -> run_project --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 08272ee..b015803 100644 --- a/setup.py +++ b/setup.py @@ -30,5 +30,5 @@ setup( "dev": dev_requires, "prod": requires, }, - entry_points={"console_scripts": ["pypi-list = pypi_list:main"]}, + entry_points={"console_scripts": ["pypi-list = pypi_list:run_project"]}, ) From 01101bcb026175f12fbf20dfa65af4706dc6b9e2 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Fri, 25 Jun 2021 08:32:15 -0500 Subject: [PATCH 10/10] =?UTF-8?q?Bump=20version:=200.3.0=20=E2=86=92=200.4?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pypi_list/__init__.py | 2 +- pyproject.toml | 2 +- setup.cfg | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pypi_list/__init__.py b/pypi_list/__init__.py index 66b1340..0cf25bd 100644 --- a/pypi_list/__init__.py +++ b/pypi_list/__init__.py @@ -36,7 +36,7 @@ from kedro.runner.sequential_runner import SequentialRunner logger = logging.getLogger(__name__) -__version__ = "0.3.0" +__version__ = "0.4.0" def get_body(packages: str) -> str: diff --git a/pyproject.toml b/pyproject.toml index 1720380..5d9642a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.kedro] package_name = "pypi_list" project_name = "Pypi List" -project_version = "0.3.0" +project_version = "0.4.0" [tool.isort] multi_line_output = 3 diff --git a/setup.cfg b/setup.cfg index 4685200..8c89724 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.0 +current_version = 0.4.0 commit = True tag = True diff --git a/setup.py b/setup.py index b015803..a836995 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ dev_requires = Path("requirements_dev.txt").read_text().split() setup( name="pypi_list", - version="0.3.0", + version="0.4.0", packages=find_packages(), install_requires=requires, extras_require={