From 260ef74d4ee1a220be545e4c490042d81e6adf78 Mon Sep 17 00:00:00 2001 From: Dave G Date: Tue, 5 Jul 2022 00:11:00 -0400 Subject: [PATCH] Add json output flag (#139) --- README.md | 16 ++++++++--- poetry.lock | 64 +++++++++++++++++++++---------------------- pyproject.toml | 2 +- rfd/api.py | 24 ++++++++++------- rfd/cli.py | 73 +++++++++++++++++++++++++++++++++++++++++--------- rfd/posts.py | 12 +++++++++ rfd/threads.py | 14 ++++++++++ 7 files changed, 148 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 98a1d03..dcaf419 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ - [View Hot Deals](#view-hot-deals) - [View and Sort Hot Deals](#view-and-sort-hot-deals) - [Search](#search) - - [Advanced](#advanced) + - [Regex](#regex) - [View Posts](#view-posts) + - [JSON Output](#json-output) - [Shell Completion](#shell-completion) - [bash](#bash) - [zsh](#zsh) @@ -47,7 +48,6 @@ If you have [brew](https://brew.sh): brew install davegallant/public/rfd ``` - ## Usage All commands open up in a [terminal pager](https://en.wikipedia.org/wiki/Terminal_pager). @@ -93,7 +93,7 @@ rfd threads --sort-by views --pages 10 rfd search 'pizza' ``` -#### Advanced +#### Regex Regular expressions can be used for search. @@ -111,6 +111,16 @@ rfd posts https://forums.redflagdeals.com/kobo-vs-kindle-2396227/ This allows for easy grepping and searching for desired expressions. +### JSON Output + +All commands support JSON output. + +For example: + +```sh +rfd threads --output json +``` + ## Shell Completion Shell completion can be enabled if using `bash` or `zsh`. diff --git a/poetry.lock b/poetry.lock index 9c76d04..55f3204 100644 --- a/poetry.lock +++ b/poetry.lock @@ -51,11 +51,11 @@ lxml = ["lxml"] [[package]] name = "certifi" -version = "2021.10.8" +version = "2022.6.15" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "cfgv" @@ -397,7 +397,7 @@ python-versions = ">=3.6" [[package]] name = "typed-ast" -version = "1.5.3" +version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" category = "dev" optional = false @@ -426,7 +426,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.14.1" +version = "20.15.1" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -487,8 +487,8 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, ] certifi = [ - {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, - {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] cfgv = [ {file = "cfgv-3.0.0-py2.py3-none-any.whl", hash = "sha256:f22b426ed59cd2ab2b54ff96608d846c33dfb8766a67f0b4a6ce130ce244414f"}, @@ -677,30 +677,30 @@ tomli = [ {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, ] typed-ast = [ - {file = "typed_ast-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ad3b48cf2b487be140072fb86feff36801487d4abb7382bb1929aaac80638ea"}, - {file = "typed_ast-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:542cd732351ba8235f20faa0fc7398946fe1a57f2cdb289e5497e1e7f48cfedb"}, - {file = "typed_ast-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc2c11ae59003d4a26dda637222d9ae924387f96acae9492df663843aefad55"}, - {file = "typed_ast-1.5.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd5df1313915dbd70eaaa88c19030b441742e8b05e6103c631c83b75e0435ccc"}, - {file = "typed_ast-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:e34f9b9e61333ecb0f7d79c21c28aa5cd63bec15cb7e1310d7d3da6ce886bc9b"}, - {file = "typed_ast-1.5.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f818c5b81966d4728fec14caa338e30a70dfc3da577984d38f97816c4b3071ec"}, - {file = "typed_ast-1.5.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3042bfc9ca118712c9809201f55355479cfcdc17449f9f8db5e744e9625c6805"}, - {file = "typed_ast-1.5.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4fff9fdcce59dc61ec1b317bdb319f8f4e6b69ebbe61193ae0a60c5f9333dc49"}, - {file = "typed_ast-1.5.3-cp36-cp36m-win_amd64.whl", hash = "sha256:8e0b8528838ffd426fea8d18bde4c73bcb4167218998cc8b9ee0a0f2bfe678a6"}, - {file = "typed_ast-1.5.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ef1d96ad05a291f5c36895d86d1375c0ee70595b90f6bb5f5fdbee749b146db"}, - {file = "typed_ast-1.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed44e81517364cb5ba367e4f68fca01fba42a7a4690d40c07886586ac267d9b9"}, - {file = "typed_ast-1.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f60d9de0d087454c91b3999a296d0c4558c1666771e3460621875021bf899af9"}, - {file = "typed_ast-1.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9e237e74fd321a55c90eee9bc5d44be976979ad38a29bbd734148295c1ce7617"}, - {file = "typed_ast-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee852185964744987609b40aee1d2eb81502ae63ee8eef614558f96a56c1902d"}, - {file = "typed_ast-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:27e46cdd01d6c3a0dd8f728b6a938a6751f7bd324817501c15fb056307f918c6"}, - {file = "typed_ast-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d64dabc6336ddc10373922a146fa2256043b3b43e61f28961caec2a5207c56d5"}, - {file = "typed_ast-1.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8cdf91b0c466a6c43f36c1964772918a2c04cfa83df8001ff32a89e357f8eb06"}, - {file = "typed_ast-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:9cc9e1457e1feb06b075c8ef8aeb046a28ec351b1958b42c7c31c989c841403a"}, - {file = "typed_ast-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e20d196815eeffb3d76b75223e8ffed124e65ee62097e4e73afb5fec6b993e7a"}, - {file = "typed_ast-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:37e5349d1d5de2f4763d534ccb26809d1c24b180a477659a12c4bde9dd677d74"}, - {file = "typed_ast-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f1a27592fac87daa4e3f16538713d705599b0a27dfe25518b80b6b017f0a6d"}, - {file = "typed_ast-1.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8831479695eadc8b5ffed06fdfb3e424adc37962a75925668deeb503f446c0a3"}, - {file = "typed_ast-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:20d5118e494478ef2d3a2702d964dae830aedd7b4d3b626d003eea526be18718"}, - {file = "typed_ast-1.5.3.tar.gz", hash = "sha256:27f25232e2dd0edfe1f019d6bfaaf11e86e657d9bdb7b0956db95f560cceb2b3"}, + {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, + {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, + {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, + {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, + {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, + {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, + {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, + {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, + {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, + {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, ] typing-extensions = [ {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, @@ -711,8 +711,8 @@ urllib3 = [ {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, ] virtualenv = [ - {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, - {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, + {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"}, + {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"}, ] wrapt = [ {file = "wrapt-1.13.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a"}, diff --git a/pyproject.toml b/pyproject.toml index e94ee11..ff46425 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "rfd" -version = "0.8.1" +version = "0.9.0" description = "view RedFlagDeals.com from the command line" authors = ["Dave Gallant "] license = "GPL-3.0-or-later" diff --git a/rfd/api.py b/rfd/api.py index 961b4a4..723d924 100644 --- a/rfd/api.py +++ b/rfd/api.py @@ -65,7 +65,7 @@ def get_posts(post): Args: post (str): either full url or postid - Yields: + Returns: list(Post): Posts """ if is_valid_url(post): @@ -81,22 +81,28 @@ def get_posts(post): total_pages = response.json().get("pager").get("total_pages") + posts = [] + for page in range(0, total_pages + 1): response = requests.get( f"{API_BASE_URL}/api/topics/{post_id}/posts?per_page=40&page={page}" ) users = create_user_map(response.json().get("users")) - posts = response.json().get("posts") + current_posts = response.json().get("posts") - for i in posts: + for _post in current_posts: # Sometimes votes is null - if i.get("votes") is not None: - calculated_score = calculate_score(i) + if _post.get("votes") is not None: + calculated_score = calculate_score(_post) else: calculated_score = 0 - yield Post( - body=strip_html(i.get("body")), - score=calculated_score, - user=users[i.get("author_id")], + posts.append( + Post( + body=strip_html(_post.get("body")), + score=calculated_score, + user=users[_post.get("author_id")], + ) ) + + return posts diff --git a/rfd/cli.py b/rfd/cli.py index 5fd56eb..a57d79b 100644 --- a/rfd/cli.py +++ b/rfd/cli.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import logging import sys +import json import click try: @@ -11,8 +12,14 @@ except ImportError: # for Python<3.8 import importlib_metadata as metadata from colorama import init from .api import get_threads, get_posts -from .threads import parse_threads, search_threads, sort_threads, generate_thread_output -from .posts import generate_posts_output +from .threads import ( + parse_threads, + search_threads, + sort_threads, + generate_thread_output, + ThreadEncoder, +) +from .posts import generate_posts_output, PostEncoder init() @@ -51,7 +58,10 @@ def cli(ctx): @cli.command(short_help="Display all posts in a thread.") @click.argument("post_id") -def posts(post_id): +@click.option( + "--output", default=None, help="Defaults to custom formatting. Other options: json" +) +def posts(post_id, output): """Iterate all pages and display all posts in a thread. post_id can be a full url or post id only @@ -63,12 +73,23 @@ def posts(post_id): """ try: - click.echo_via_pager(generate_posts_output(get_posts(post=post_id))) + if output == "json": + click.echo_via_pager( + json.dumps( + get_posts(post=post_id), + cls=PostEncoder, + indent=2, + sort_keys=True, + ) + ) + else: + click.echo_via_pager(generate_posts_output(get_posts(post=post_id))) + except ValueError: click.echo("Invalid post id.") sys.exit(1) - except AttributeError: - click.echo("The RFD API did not return the expected data.") + except AttributeError as err: + click.echo("The RFD API did not return the expected data. %s", err) sys.exit(1) @@ -76,7 +97,10 @@ def posts(post_id): @click.option("--forum-id", default=9, help="The forum id number") @click.option("--pages", default=1, help="Number of pages to show. Defaults to 1.") @click.option("--sort-by", default=None, help="Sort threads by") -def threads(forum_id, pages, sort_by): +@click.option( + "--output", default=None, help="Defaults to custom formatting. Other options: json" +) +def threads(forum_id, pages, sort_by, output): """Display threads in the specified forum id. Defaults to 9 (hot deals). Popular forum ids: @@ -96,7 +120,18 @@ def threads(forum_id, pages, sort_by): _threads = sort_threads( parse_threads(get_threads(forum_id, pages)), sort_by=sort_by ) - click.echo_via_pager(generate_thread_output(_threads)) + if output == "json": + + click.echo_via_pager( + json.dumps( + sort_threads(_threads, sort_by=sort_by), + cls=ThreadEncoder, + indent=2, + sort_keys=True, + ) + ) + else: + click.echo_via_pager(generate_thread_output(_threads)) @cli.command(short_help="Search deals based on a regular expression.") @@ -105,8 +140,11 @@ def threads(forum_id, pages, sort_by): "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)." ) @click.option("--sort-by", default=None, help="Sort threads by") +@click.option( + "--output", default=None, help="Defaults to custom formatting. Other options: json" +) @click.argument("regex") -def search(pages, forum_id, sort_by, regex): +def search(pages, forum_id, sort_by, output, regex): """Search deals based on regex. Popular forum ids: @@ -129,6 +167,17 @@ def search(pages, forum_id, sort_by, regex): _threads = parse_threads(get_threads(forum_id, pages=pages)) for thread in search_threads(threads=_threads, regex=regex): matched_threads.append(thread) - click.echo_via_pager( - generate_thread_output(sort_threads(matched_threads, sort_by=sort_by)) - ) + + if output == "json": + click.echo_via_pager( + json.dumps( + sort_threads(matched_threads, sort_by=sort_by), + indent=2, + sort_keys=True, + cls=ThreadEncoder, + ) + ) + else: + click.echo_via_pager( + generate_thread_output(sort_threads(matched_threads, sort_by=sort_by)) + ) diff --git a/rfd/posts.py b/rfd/posts.py index 5cf19d4..1073faa 100644 --- a/rfd/posts.py +++ b/rfd/posts.py @@ -1,5 +1,6 @@ # pylint: disable=old-style-class import os +import json from colorama import Fore, Style from .scores import get_vote_color @@ -11,6 +12,17 @@ class Post: self.user = user +class PostEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, Post): + return dict( + body=o.body, + score=o.score, + user=o.user, + ) + return json.JSONEncoder.default(self, o) + + def get_terminal_width(): _, columns = os.popen("stty size", "r").read().split() return int(columns) diff --git a/rfd/threads.py b/rfd/threads.py index 91656f4..eb3d55b 100644 --- a/rfd/threads.py +++ b/rfd/threads.py @@ -1,4 +1,5 @@ import re +import json from colorama import Fore, Style from . import API_BASE_URL from .scores import calculate_score, get_vote_color @@ -16,6 +17,19 @@ class Thread: return f"Thread({self.title})" +class ThreadEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, Thread): + return dict( + dealer_name=o.dealer_name, + score=o.score, + title=o.title, + url=o.url, + views=o.views, + ) + return json.JSONEncoder.default(self, o) + + def build_web_path(slug): return f"{API_BASE_URL}{slug}"