Add json output flag (#139)

README.md

@@ -12,8 +12,9 @@
   - [View Hot Deals](#view-hot-deals)
   - [View and Sort Hot Deals](#view-and-sort-hot-deals)
   - [Search](#search)
-    - [Advanced](#advanced)
+    - [Regex](#regex)
   - [View Posts](#view-posts)
+  - [JSON Output](#json-output)
 - [Shell Completion](#shell-completion)
   - [bash](#bash)
   - [zsh](#zsh)
@@ -47,7 +48,6 @@ If you have [brew](https://brew.sh):
 brew install davegallant/public/rfd
 ```
 
-
 ## Usage
 
 All commands open up in a [terminal pager](https://en.wikipedia.org/wiki/Terminal_pager).
@@ -93,7 +93,7 @@ rfd threads --sort-by views --pages 10
 rfd search 'pizza'
 ```
 
-#### Advanced
+#### Regex
 
 Regular expressions can be used for search.
 
@@ -111,6 +111,16 @@ rfd posts https://forums.redflagdeals.com/kobo-vs-kindle-2396227/
 
 This allows for easy grepping and searching for desired expressions.
 
+### JSON Output
+
+All commands support JSON output.
+
+For example:
+
+```sh
+rfd threads --output json
+```
+
 ## Shell Completion
 
 Shell completion can be enabled if using `bash` or `zsh`.

poetry.lock

@@ -51,11 +51,11 @@ lxml = ["lxml"]
 
 [[package]]
 name = "certifi"
-version = "2021.10.8"
+version = "2022.6.15"
 description = "Python package for providing Mozilla's CA Bundle."
 category = "main"
 optional = false
-python-versions = "*"
+python-versions = ">=3.6"
 
 [[package]]
 name = "cfgv"
@@ -397,7 +397,7 @@ python-versions = ">=3.6"
 
 [[package]]
 name = "typed-ast"
-version = "1.5.3"
+version = "1.5.4"
 description = "a fork of Python 2 and 3 ast modules with type comment support"
 category = "dev"
 optional = false
@@ -426,7 +426,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 
 [[package]]
 name = "virtualenv"
-version = "20.14.1"
+version = "20.15.1"
 description = "Virtual Python Environment builder"
 category = "dev"
 optional = false
@@ -487,8 +487,8 @@ beautifulsoup4 = [
     {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"},
 ]
 certifi = [
-    {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
+    {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"},
-    {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
+    {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"},
 ]
 cfgv = [
     {file = "cfgv-3.0.0-py2.py3-none-any.whl", hash = "sha256:f22b426ed59cd2ab2b54ff96608d846c33dfb8766a67f0b4a6ce130ce244414f"},
@@ -677,30 +677,30 @@ tomli = [
     {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"},
 ]
 typed-ast = [
-    {file = "typed_ast-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ad3b48cf2b487be140072fb86feff36801487d4abb7382bb1929aaac80638ea"},
+    {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"},
-    {file = "typed_ast-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:542cd732351ba8235f20faa0fc7398946fe1a57f2cdb289e5497e1e7f48cfedb"},
+    {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"},
-    {file = "typed_ast-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc2c11ae59003d4a26dda637222d9ae924387f96acae9492df663843aefad55"},
+    {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"},
-    {file = "typed_ast-1.5.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd5df1313915dbd70eaaa88c19030b441742e8b05e6103c631c83b75e0435ccc"},
+    {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"},
-    {file = "typed_ast-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:e34f9b9e61333ecb0f7d79c21c28aa5cd63bec15cb7e1310d7d3da6ce886bc9b"},
+    {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"},
-    {file = "typed_ast-1.5.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f818c5b81966d4728fec14caa338e30a70dfc3da577984d38f97816c4b3071ec"},
+    {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"},
-    {file = "typed_ast-1.5.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3042bfc9ca118712c9809201f55355479cfcdc17449f9f8db5e744e9625c6805"},
+    {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"},
-    {file = "typed_ast-1.5.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4fff9fdcce59dc61ec1b317bdb319f8f4e6b69ebbe61193ae0a60c5f9333dc49"},
+    {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"},
-    {file = "typed_ast-1.5.3-cp36-cp36m-win_amd64.whl", hash = "sha256:8e0b8528838ffd426fea8d18bde4c73bcb4167218998cc8b9ee0a0f2bfe678a6"},
+    {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"},
-    {file = "typed_ast-1.5.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ef1d96ad05a291f5c36895d86d1375c0ee70595b90f6bb5f5fdbee749b146db"},
+    {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"},
-    {file = "typed_ast-1.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed44e81517364cb5ba367e4f68fca01fba42a7a4690d40c07886586ac267d9b9"},
+    {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"},
-    {file = "typed_ast-1.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f60d9de0d087454c91b3999a296d0c4558c1666771e3460621875021bf899af9"},
+    {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"},
-    {file = "typed_ast-1.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9e237e74fd321a55c90eee9bc5d44be976979ad38a29bbd734148295c1ce7617"},
+    {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"},
-    {file = "typed_ast-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee852185964744987609b40aee1d2eb81502ae63ee8eef614558f96a56c1902d"},
+    {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"},
-    {file = "typed_ast-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:27e46cdd01d6c3a0dd8f728b6a938a6751f7bd324817501c15fb056307f918c6"},
+    {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"},
-    {file = "typed_ast-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d64dabc6336ddc10373922a146fa2256043b3b43e61f28961caec2a5207c56d5"},
+    {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"},
-    {file = "typed_ast-1.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8cdf91b0c466a6c43f36c1964772918a2c04cfa83df8001ff32a89e357f8eb06"},
+    {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"},
-    {file = "typed_ast-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:9cc9e1457e1feb06b075c8ef8aeb046a28ec351b1958b42c7c31c989c841403a"},
+    {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"},
-    {file = "typed_ast-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e20d196815eeffb3d76b75223e8ffed124e65ee62097e4e73afb5fec6b993e7a"},
+    {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"},
-    {file = "typed_ast-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:37e5349d1d5de2f4763d534ccb26809d1c24b180a477659a12c4bde9dd677d74"},
+    {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"},
-    {file = "typed_ast-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f1a27592fac87daa4e3f16538713d705599b0a27dfe25518b80b6b017f0a6d"},
+    {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"},
-    {file = "typed_ast-1.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8831479695eadc8b5ffed06fdfb3e424adc37962a75925668deeb503f446c0a3"},
+    {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"},
-    {file = "typed_ast-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:20d5118e494478ef2d3a2702d964dae830aedd7b4d3b626d003eea526be18718"},
+    {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"},
-    {file = "typed_ast-1.5.3.tar.gz", hash = "sha256:27f25232e2dd0edfe1f019d6bfaaf11e86e657d9bdb7b0956db95f560cceb2b3"},
+    {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"},
 ]
 typing-extensions = [
     {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"},
@@ -711,8 +711,8 @@ urllib3 = [
     {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
 ]
 virtualenv = [
-    {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"},
+    {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"},
-    {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"},
+    {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"},
 ]
 wrapt = [
     {file = "wrapt-1.13.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a"},

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rfd"
-version = "0.8.1"
+version = "0.9.0"
 description = "view RedFlagDeals.com from the command line"
 authors = ["Dave Gallant <davegallant@gmail.com>"]
 license = "GPL-3.0-or-later"

rfd/api.py

@@ -65,7 +65,7 @@ def get_posts(post):
     Args:
         post (str): either full url or postid
 
-    Yields:
+    Returns:
         list(Post): Posts
     """
     if is_valid_url(post):
@@ -81,22 +81,28 @@ def get_posts(post):
 
     total_pages = response.json().get("pager").get("total_pages")
 
+    posts = []
+
     for page in range(0, total_pages + 1):
         response = requests.get(
             f"{API_BASE_URL}/api/topics/{post_id}/posts?per_page=40&page={page}"
         )
         users = create_user_map(response.json().get("users"))
 
-        posts = response.json().get("posts")
+        current_posts = response.json().get("posts")
 
-        for i in posts:
+        for _post in current_posts:
             # Sometimes votes is null
-            if i.get("votes") is not None:
+            if _post.get("votes") is not None:
-                calculated_score = calculate_score(i)
+                calculated_score = calculate_score(_post)
             else:
                 calculated_score = 0
-            yield Post(
+            posts.append(
-                body=strip_html(i.get("body")),
+                Post(
-                score=calculated_score,
+                    body=strip_html(_post.get("body")),
-                user=users[i.get("author_id")],
+                    score=calculated_score,
+                    user=users[_post.get("author_id")],
+                )
             )
+
+    return posts

rfd/cli.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import logging
 import sys
+import json
 import click
 
 try:
@@ -11,8 +12,14 @@ except ImportError:  # for Python<3.8
     import importlib_metadata as metadata
 from colorama import init
 from .api import get_threads, get_posts
-from .threads import parse_threads, search_threads, sort_threads, generate_thread_output
+from .threads import (
-from .posts import generate_posts_output
+    parse_threads,
+    search_threads,
+    sort_threads,
+    generate_thread_output,
+    ThreadEncoder,
+)
+from .posts import generate_posts_output, PostEncoder
 
 
 init()
@@ -51,7 +58,10 @@ def cli(ctx):
 
 @cli.command(short_help="Display all posts in a thread.")
 @click.argument("post_id")
-def posts(post_id):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def posts(post_id, output):
     """Iterate all pages and display all posts in a thread.
 
     post_id can be a full url or post id only
@@ -63,12 +73,23 @@ def posts(post_id):
     """
 
     try:
-        click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+        if output == "json":
+            click.echo_via_pager(
+                json.dumps(
+                    get_posts(post=post_id),
+                    cls=PostEncoder,
+                    indent=2,
+                    sort_keys=True,
+                )
+            )
+        else:
+            click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
+
     except ValueError:
         click.echo("Invalid post id.")
         sys.exit(1)
-    except AttributeError:
+    except AttributeError as err:
-        click.echo("The RFD API did not return the expected data.")
+        click.echo("The RFD API did not return the expected data. %s", err)
         sys.exit(1)
 
 
@@ -76,7 +97,10 @@ def posts(post_id):
 @click.option("--forum-id", default=9, help="The forum id number")
 @click.option("--pages", default=1, help="Number of pages to show. Defaults to 1.")
 @click.option("--sort-by", default=None, help="Sort threads by")
-def threads(forum_id, pages, sort_by):
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
+def threads(forum_id, pages, sort_by, output):
     """Display threads in the specified forum id. Defaults to 9 (hot deals).
 
     Popular forum ids:
@@ -96,7 +120,18 @@ def threads(forum_id, pages, sort_by):
     _threads = sort_threads(
         parse_threads(get_threads(forum_id, pages)), sort_by=sort_by
     )
-    click.echo_via_pager(generate_thread_output(_threads))
+    if output == "json":
+
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(_threads, sort_by=sort_by),
+                cls=ThreadEncoder,
+                indent=2,
+                sort_keys=True,
+            )
+        )
+    else:
+        click.echo_via_pager(generate_thread_output(_threads))
 
 
 @cli.command(short_help="Search deals based on a regular expression.")
@@ -105,8 +140,11 @@ def threads(forum_id, pages, sort_by):
     "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)."
 )
 @click.option("--sort-by", default=None, help="Sort threads by")
+@click.option(
+    "--output", default=None, help="Defaults to custom formatting. Other options: json"
+)
 @click.argument("regex")
-def search(pages, forum_id, sort_by, regex):
+def search(pages, forum_id, sort_by, output, regex):
     """Search deals based on regex.
 
     Popular forum ids:
@@ -129,6 +167,17 @@ def search(pages, forum_id, sort_by, regex):
     _threads = parse_threads(get_threads(forum_id, pages=pages))
     for thread in search_threads(threads=_threads, regex=regex):
         matched_threads.append(thread)
-    click.echo_via_pager(
+
-        generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
+    if output == "json":
-    )
+        click.echo_via_pager(
+            json.dumps(
+                sort_threads(matched_threads, sort_by=sort_by),
+                indent=2,
+                sort_keys=True,
+                cls=ThreadEncoder,
+            )
+        )
+    else:
+        click.echo_via_pager(
+            generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
+        )

rfd/posts.py

@@ -1,5 +1,6 @@
 # pylint: disable=old-style-class
 import os
+import json
 from colorama import Fore, Style
 from .scores import get_vote_color
 
@@ -11,6 +12,17 @@ class Post:
         self.user = user
 
 
+class PostEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Post):
+            return dict(
+                body=o.body,
+                score=o.score,
+                user=o.user,
+            )
+        return json.JSONEncoder.default(self, o)
+
+
 def get_terminal_width():
     _, columns = os.popen("stty size", "r").read().split()
     return int(columns)

rfd/threads.py

@@ -1,4 +1,5 @@
 import re
+import json
 from colorama import Fore, Style
 from . import API_BASE_URL
 from .scores import calculate_score, get_vote_color
@@ -16,6 +17,19 @@ class Thread:
         return f"Thread({self.title})"
 
 
+class ThreadEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Thread):
+            return dict(
+                dealer_name=o.dealer_name,
+                score=o.score,
+                title=o.title,
+                url=o.url,
+                views=o.views,
+            )
+        return json.JSONEncoder.default(self, o)
+
+
 def build_web_path(slug):
     return f"{API_BASE_URL}{slug}"