Compare commits
10 commits
06a80e1218
...
01101bcb02
| Author | SHA1 | Date | |
|---|---|---|---|
|
01101bcb02 | ||
|
|
9e2abc427b | ||
|
|
355034216c | ||
|
|
89d29fbc5d | ||
|
|
e58af1a092 | ||
|
|
39f672cb21 | ||
|
|
6ecefb482f | ||
|
|
f7eb84c03b | ||
|
|
65241d7751 | ||
|
|
dc7511ca9a |
9 changed files with 83 additions and 15 deletions
2
.github/workflows/deploy-pypi-list.yml
vendored
2
.github/workflows/deploy-pypi-list.yml
vendored
|
|
@ -16,7 +16,7 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
|
|
||||||
- name: Setup Python
|
- name: Setup Python
|
||||||
uses: actions/setup-python@v2.2.2
|
uses: actions/setup-python@v2.2.2
|
||||||
with:
|
with:
|
||||||
|
|
|
||||||
|
|
@ -39,3 +39,11 @@ repos:
|
||||||
hooks:
|
hooks:
|
||||||
- id: mypy
|
- id: mypy
|
||||||
exclude: tests/
|
exclude: tests/
|
||||||
|
- repo: https://github.com/econchick/interrogate
|
||||||
|
rev: '1.2.0'
|
||||||
|
hooks:
|
||||||
|
- id: interrogate
|
||||||
|
description: check for docstring coverage
|
||||||
|
entry: interrogate -f 100 -iIvv
|
||||||
|
types: [python]
|
||||||
|
exclude: docs
|
||||||
|
|
|
||||||
|
|
@ -31,3 +31,11 @@ This pipeline run daily at `0:0` to generate a `packages.json`,
|
||||||
``` bash
|
``` bash
|
||||||
pip install -e ".[dev]"
|
pip install -e ".[dev]"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This project includes a `pre-commit-config`, this ensures that each commit made
|
||||||
|
to the project passes all checks that are also ran in CI. To setup pre-commit
|
||||||
|
in your development environment run the follow command.
|
||||||
|
|
||||||
|
``` bash
|
||||||
|
pre-commit install
|
||||||
|
```
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ pypi-list
|
||||||
Listing python packages from pypi, and finding available single word packages.
|
Listing python packages from pypi, and finding available single word packages.
|
||||||
|
|
||||||
|
|
||||||
## Run the Pipeline
|
## Run the Pipeline at the command line
|
||||||
|
|
||||||
``` bash
|
``` bash
|
||||||
# Run with existing package data
|
# Run with existing package data
|
||||||
|
|
@ -13,8 +13,19 @@ python pypi-list.py
|
||||||
# Full run
|
# Full run
|
||||||
python pypi-list.py --full
|
python pypi-list.py --full
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Run the Pipeline with a python repl
|
||||||
|
|
||||||
|
``` python
|
||||||
|
from pypi_list import run_project
|
||||||
|
|
||||||
|
run_project() # run local datasets only
|
||||||
|
run_project(full=True) # run full pipeline including network requests
|
||||||
|
```
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from kedro.extras.datasets.json import JSONDataSet
|
from kedro.extras.datasets.json import JSONDataSet
|
||||||
|
|
@ -25,10 +36,11 @@ from kedro.runner.sequential_runner import SequentialRunner
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
__version__ = "0.2.0"
|
__version__ = "0.4.0"
|
||||||
|
|
||||||
|
|
||||||
def get_body(packages):
|
def get_body(packages: str) -> str:
|
||||||
|
"""Get the body tag from the full page html."""
|
||||||
|
|
||||||
tag = "<body>\n"
|
tag = "<body>\n"
|
||||||
index = packages.find(tag) + len(tag)
|
index = packages.find(tag) + len(tag)
|
||||||
|
|
@ -37,7 +49,7 @@ def get_body(packages):
|
||||||
return packages
|
return packages
|
||||||
|
|
||||||
|
|
||||||
def remove_html_tags(text):
|
def remove_html_tags(text: str) -> List[str]:
|
||||||
"""Remove html tags from a string"""
|
"""Remove html tags from a string"""
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
|
@ -135,11 +147,36 @@ catalog = DataCatalog(
|
||||||
runner = SequentialRunner()
|
runner = SequentialRunner()
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def run_project(full: Optional[bool] = None):
|
||||||
|
"""
|
||||||
|
Run the project.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
--------
|
||||||
|
full : bool
|
||||||
|
runs the full pipeline if True
|
||||||
|
skips network calls if False
|
||||||
|
checks sys.arv for --full if None
|
||||||
|
|
||||||
|
Returns
|
||||||
|
--------
|
||||||
|
None
|
||||||
|
|
||||||
|
Examples
|
||||||
|
--------
|
||||||
|
>>> from pypi_list import run_project
|
||||||
|
>>> run_project() # run local datasets only
|
||||||
|
>>> run_project(full=True) # run full pipeline including network requests
|
||||||
|
|
||||||
|
"""
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
if "--full" in sys.argv:
|
if "--full" in sys.argv and full is None:
|
||||||
|
full = True
|
||||||
|
|
||||||
|
if full:
|
||||||
runner.run(pipeline, catalog)
|
runner.run(pipeline, catalog)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
runner.run(
|
runner.run(
|
||||||
Pipeline([node for node in pipeline.nodes if "raw" not in node.name]),
|
Pipeline([node for node in pipeline.nodes if "raw" not in node.name]),
|
||||||
|
|
@ -148,4 +185,4 @@ def main():
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
run_project()
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
[tool.kedro]
|
[tool.kedro]
|
||||||
package_name = "pypi_list"
|
package_name = "pypi_list"
|
||||||
project_name = "Pypi List"
|
project_name = "Pypi List"
|
||||||
project_version = "0.0.2"
|
project_version = "0.4.0"
|
||||||
|
|
||||||
[tool.isort]
|
[tool.isort]
|
||||||
multi_line_output = 3
|
multi_line_output = 3
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,2 @@
|
||||||
kedro
|
kedro
|
||||||
requests
|
requests
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
[bumpversion]
|
[bumpversion]
|
||||||
current_version = 0.2.0
|
current_version = 0.4.0
|
||||||
commit = True
|
commit = True
|
||||||
tag = True
|
tag = True
|
||||||
|
|
||||||
|
|
|
||||||
22
setup.py
22
setup.py
|
|
@ -1,12 +1,28 @@
|
||||||
from setuptools import setup, find_packages
|
"""
|
||||||
|
pypi-list uses setup tools for packaging.
|
||||||
|
|
||||||
|
To Build pypi-list as a Python package
|
||||||
|
|
||||||
|
$ python setup.py sdist bdist_wheel
|
||||||
|
|
||||||
|
Regular install
|
||||||
|
|
||||||
|
$ pip install -e .
|
||||||
|
|
||||||
|
To setup local Development
|
||||||
|
|
||||||
|
$ pip install -e ".[dev]"
|
||||||
|
"""
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
requires = Path("requirements.txt").read_text().split()
|
requires = Path("requirements.txt").read_text().split()
|
||||||
dev_requires = Path("requirements_dev.txt").read_text().split()
|
dev_requires = Path("requirements_dev.txt").read_text().split()
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="pypi_list",
|
name="pypi_list",
|
||||||
version="0.2.0",
|
version="0.4.0",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
install_requires=requires,
|
install_requires=requires,
|
||||||
extras_require={
|
extras_require={
|
||||||
|
|
@ -14,5 +30,5 @@ setup(
|
||||||
"dev": dev_requires,
|
"dev": dev_requires,
|
||||||
"prod": requires,
|
"prod": requires,
|
||||||
},
|
},
|
||||||
entry_points={"console_scripts": ["pypi-list = pypi_list:main"]},
|
entry_points={"console_scripts": ["pypi-list = pypi_list:run_project"]},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,6 @@
|
||||||
<body>
|
<body>
|
||||||
<h1>pypi-list</h1>
|
<h1>pypi-list</h1>
|
||||||
Checkout the available pypi packages that are single words in <a href='./available.json'>available.json</a>
|
Checkout the available pypi packages that are single words in <a href='./available.json'>available.json</a>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue