Compare commits
No commits in common. "01101bcb026175f12fbf20dfa65af4706dc6b9e2" and "06a80e1218dc551fda3ddf47c1c821b00a623664" have entirely different histories.
01101bcb02
...
06a80e1218
9 changed files with 15 additions and 83 deletions
2
.github/workflows/deploy-pypi-list.yml
vendored
2
.github/workflows/deploy-pypi-list.yml
vendored
|
|
@ -16,7 +16,7 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
|
|
||||||
- name: Setup Python
|
- name: Setup Python
|
||||||
uses: actions/setup-python@v2.2.2
|
uses: actions/setup-python@v2.2.2
|
||||||
with:
|
with:
|
||||||
|
|
|
||||||
|
|
@ -39,11 +39,3 @@ repos:
|
||||||
hooks:
|
hooks:
|
||||||
- id: mypy
|
- id: mypy
|
||||||
exclude: tests/
|
exclude: tests/
|
||||||
- repo: https://github.com/econchick/interrogate
|
|
||||||
rev: '1.2.0'
|
|
||||||
hooks:
|
|
||||||
- id: interrogate
|
|
||||||
description: check for docstring coverage
|
|
||||||
entry: interrogate -f 100 -iIvv
|
|
||||||
types: [python]
|
|
||||||
exclude: docs
|
|
||||||
|
|
|
||||||
|
|
@ -31,11 +31,3 @@ This pipeline run daily at `0:0` to generate a `packages.json`,
|
||||||
``` bash
|
``` bash
|
||||||
pip install -e ".[dev]"
|
pip install -e ".[dev]"
|
||||||
```
|
```
|
||||||
|
|
||||||
This project includes a `pre-commit-config`, this ensures that each commit made
|
|
||||||
to the project passes all checks that are also ran in CI. To setup pre-commit
|
|
||||||
in your development environment run the follow command.
|
|
||||||
|
|
||||||
``` bash
|
|
||||||
pre-commit install
|
|
||||||
```
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ pypi-list
|
||||||
Listing python packages from pypi, and finding available single word packages.
|
Listing python packages from pypi, and finding available single word packages.
|
||||||
|
|
||||||
|
|
||||||
## Run the Pipeline at the command line
|
## Run the Pipeline
|
||||||
|
|
||||||
``` bash
|
``` bash
|
||||||
# Run with existing package data
|
# Run with existing package data
|
||||||
|
|
@ -13,19 +13,8 @@ python pypi-list.py
|
||||||
# Full run
|
# Full run
|
||||||
python pypi-list.py --full
|
python pypi-list.py --full
|
||||||
```
|
```
|
||||||
|
|
||||||
## Run the Pipeline with a python repl
|
|
||||||
|
|
||||||
``` python
|
|
||||||
from pypi_list import run_project
|
|
||||||
|
|
||||||
run_project() # run local datasets only
|
|
||||||
run_project(full=True) # run full pipeline including network requests
|
|
||||||
```
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from kedro.extras.datasets.json import JSONDataSet
|
from kedro.extras.datasets.json import JSONDataSet
|
||||||
|
|
@ -36,11 +25,10 @@ from kedro.runner.sequential_runner import SequentialRunner
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
__version__ = "0.4.0"
|
__version__ = "0.2.0"
|
||||||
|
|
||||||
|
|
||||||
def get_body(packages: str) -> str:
|
def get_body(packages):
|
||||||
"""Get the body tag from the full page html."""
|
|
||||||
|
|
||||||
tag = "<body>\n"
|
tag = "<body>\n"
|
||||||
index = packages.find(tag) + len(tag)
|
index = packages.find(tag) + len(tag)
|
||||||
|
|
@ -49,7 +37,7 @@ def get_body(packages: str) -> str:
|
||||||
return packages
|
return packages
|
||||||
|
|
||||||
|
|
||||||
def remove_html_tags(text: str) -> List[str]:
|
def remove_html_tags(text):
|
||||||
"""Remove html tags from a string"""
|
"""Remove html tags from a string"""
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
|
@ -147,36 +135,11 @@ catalog = DataCatalog(
|
||||||
runner = SequentialRunner()
|
runner = SequentialRunner()
|
||||||
|
|
||||||
|
|
||||||
def run_project(full: Optional[bool] = None):
|
def main():
|
||||||
"""
|
|
||||||
Run the project.
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
--------
|
|
||||||
full : bool
|
|
||||||
runs the full pipeline if True
|
|
||||||
skips network calls if False
|
|
||||||
checks sys.arv for --full if None
|
|
||||||
|
|
||||||
Returns
|
|
||||||
--------
|
|
||||||
None
|
|
||||||
|
|
||||||
Examples
|
|
||||||
--------
|
|
||||||
>>> from pypi_list import run_project
|
|
||||||
>>> run_project() # run local datasets only
|
|
||||||
>>> run_project(full=True) # run full pipeline including network requests
|
|
||||||
|
|
||||||
"""
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
if "--full" in sys.argv and full is None:
|
if "--full" in sys.argv:
|
||||||
full = True
|
|
||||||
|
|
||||||
if full:
|
|
||||||
runner.run(pipeline, catalog)
|
runner.run(pipeline, catalog)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
runner.run(
|
runner.run(
|
||||||
Pipeline([node for node in pipeline.nodes if "raw" not in node.name]),
|
Pipeline([node for node in pipeline.nodes if "raw" not in node.name]),
|
||||||
|
|
@ -185,4 +148,4 @@ def run_project(full: Optional[bool] = None):
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
run_project()
|
main()
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
[tool.kedro]
|
[tool.kedro]
|
||||||
package_name = "pypi_list"
|
package_name = "pypi_list"
|
||||||
project_name = "Pypi List"
|
project_name = "Pypi List"
|
||||||
project_version = "0.4.0"
|
project_version = "0.0.2"
|
||||||
|
|
||||||
[tool.isort]
|
[tool.isort]
|
||||||
multi_line_output = 3
|
multi_line_output = 3
|
||||||
|
|
|
||||||
|
|
@ -1,2 +1,3 @@
|
||||||
kedro
|
kedro
|
||||||
requests
|
requests
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
[bumpversion]
|
[bumpversion]
|
||||||
current_version = 0.4.0
|
current_version = 0.2.0
|
||||||
commit = True
|
commit = True
|
||||||
tag = True
|
tag = True
|
||||||
|
|
||||||
|
|
|
||||||
22
setup.py
22
setup.py
|
|
@ -1,28 +1,12 @@
|
||||||
"""
|
from setuptools import setup, find_packages
|
||||||
pypi-list uses setup tools for packaging.
|
|
||||||
|
|
||||||
To Build pypi-list as a Python package
|
|
||||||
|
|
||||||
$ python setup.py sdist bdist_wheel
|
|
||||||
|
|
||||||
Regular install
|
|
||||||
|
|
||||||
$ pip install -e .
|
|
||||||
|
|
||||||
To setup local Development
|
|
||||||
|
|
||||||
$ pip install -e ".[dev]"
|
|
||||||
"""
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from setuptools import find_packages, setup
|
|
||||||
|
|
||||||
requires = Path("requirements.txt").read_text().split()
|
requires = Path("requirements.txt").read_text().split()
|
||||||
dev_requires = Path("requirements_dev.txt").read_text().split()
|
dev_requires = Path("requirements_dev.txt").read_text().split()
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="pypi_list",
|
name="pypi_list",
|
||||||
version="0.4.0",
|
version="0.2.0",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
install_requires=requires,
|
install_requires=requires,
|
||||||
extras_require={
|
extras_require={
|
||||||
|
|
@ -30,5 +14,5 @@ setup(
|
||||||
"dev": dev_requires,
|
"dev": dev_requires,
|
||||||
"prod": requires,
|
"prod": requires,
|
||||||
},
|
},
|
||||||
entry_points={"console_scripts": ["pypi-list = pypi_list:run_project"]},
|
entry_points={"console_scripts": ["pypi-list = pypi_list:main"]},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,6 @@
|
||||||
<body>
|
<body>
|
||||||
<h1>pypi-list</h1>
|
<h1>pypi-list</h1>
|
||||||
Checkout the available pypi packages that are single words in <a href='./available.json'>available.json</a>
|
Checkout the available pypi packages that are single words in <a href='./available.json'>available.json</a>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue