Bump to 0.6.0 (#82)

* Set default of threads command to 1 page and 40 threads

* Echo posts in a pager

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,8 +1 @@
-
+###### Motivation for this change
-*Description of changes:*
-
-*Checklist:*
-
-- [ ] Write unit tests
-- [ ] `make pr` passes
-- [ ] Write documentation

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,27 @@
+name: "Code scanning - action"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 4 * * 1'
+
+jobs:
+  CodeQL-Build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.gitignore

@@ -88,3 +88,5 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
+
+tmp/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v2.5.0
     hooks:
     -   id: check-added-large-files
     -   id: check-ast

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-dist: xenial
+dist: bionic
 sudo: false
 cache: false
 stages:
@@ -18,15 +18,14 @@ script:
 jobs:
   include:
     - python: "2.7"
-
     - python: "3.5"
-
     - python: "3.6"
-
     - python: "3.7"
+    - python: "3.8"
+    - python: "3.9-dev"
 
     - stage: deploy
-      python: "3.7"
+      python: "3.8"
       deploy:
         on:
           repo: davegallant/rfd

Makefile

@@ -1,35 +1,52 @@
 
-SRC:=rfd
+SRC := rfd
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
 
-.PHONY: build
+
+ifeq ($(origin .RECIPEPREFIX), undefined)
+  $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
+endif
+.RECIPEPREFIX = >
+
+## build: Build a tar.gz of the python package
 build:
-	rm -rf dist/
+> rm -rf dist/
-	python setup.py sdist
+> python setup.py sdist
-
+.PHONY: build
-.PHONY: push_test
-push_test:
-	twine upload -r testpypi dist/*.tar.gz
-
-.PHONY: push_prod
-push_prod:
-	twine upload dist/*.tar.gz
 
+## precommit: Run all pre-commit hooks
+precommit:
+> pre-commit run \
+ --all-files \
+ --show-diff-on-failure
 .PHONY: precommit
-precommit: ## Run pre-commit
-	pre-commit run \
-	--all-files \
-	--show-diff-on-failure
 
-.PHONY: lint
+## lint: Run static analysis on the code
 lint:
-	pylint $(SRC)
+> pylint $(SRC)
+.PHONY: lint
 
-.PHONY: test
+## test: Run all unit tests
 test:
-	pytest -v
+> pytest -vvv tests
+.PHONY: test
 
-.PHONY: pr
+## pr: Run pre-commit, lint and test
 pr: precommit lint test
+.PHONY: pr
 
-.PHONY: ci
 ci: lint test
+.PHONY: ci
+
+## help: Print this help message
+help:
+> @echo "Usage:"
+> @echo
+> @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /' | sort
+> @echo
+.PHONY: help

README.md

@@ -5,10 +5,11 @@ Hot deals on the command line.
 [![Build Status](https://travis-ci.org/davegallant/rfd.svg?branch=master)](https://travis-ci.org/davegallant/rfd)
 [![PyPI version](https://badge.fury.io/py/rfd.svg)](https://badge.fury.io/py/rfd)
 [![Dependabot](https://badgen.net/badge/Dependabot/enabled/green?icon=dependabot)](https://dependabot.com/)
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/davegallant/rfd.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/davegallant/rfd/alerts/)
+[![Downloads](https://pepy.tech/badge/rfd)](https://pepy.tech/project/rfd)
-[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/davegallant/rfd.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/davegallant/rfd/context:python)
 
-## Installation
+![screenshot](https://user-images.githubusercontent.com/4519234/85969861-e10a4100-b996-11ea-9a31-6203322c60ee.png)
+
+## Install
 
 ```bash
 pip install rfd
@@ -16,30 +17,77 @@ pip install rfd
 
 ## Usage
 
-![rfd_demo_gif](https://user-images.githubusercontent.com/4519234/64501455-64836600-d28f-11e9-8381-3fbfda910230.gif)
+```shell
+Usage: rfd [OPTIONS] COMMAND [ARGS]...
 
+  CLI for https://forums.redflagdeals.com
 
-### threads
+Options:
-```bash
+  -v, --version
-rfd threads [--forum-id 9] [--limit 10]
+  --help         Show this message and exit.
+
+Commands:
+  posts    Display all posts in a thread.
+  search   Search deals based on a regular expression.
+  threads  Displays threads in the forum. Defaults to hot deals.
 ```
 
-### search
+## Examples
-```bash
+
-rfd search pizza [--num-pages 100]
+All commands open up in a pager.
+
+Search can be done using `/`.
+
+Close pager with `q`.
+
+### View Hot Deals
+
+```console
+$ rfd threads
 ```
 
-## Tab Completion
+### View and Sort Hot Deals
+
+```console
+$ rfd threads --sort-by score
+```
+
+```console
+$ rfd threads --sort-by views --pages 10
+```
+
+### Simple Search
+
+```console
+$ rfd search 'pizza'
+```
+
+### Advanced Search
+
+Regular expressions can be used for search.
+
+```console
+$ rfd search '(coffee|starbucks)' --pages 10 --sort-by views
+```
+
+### View Posts
+
+```console
+$ rfd posts https://forums.redflagdeals.com/kobo-vs-kindle-2396227/
+```
+
+## Shell Completion
+
+Completion can be enabled if using `bash` or `zsh`.
 
 ### bash
 
-```bash
+```console
-echo 'eval "$(_RFD_COMPLETE=source rfd)"' >> ~/.profile
+$ echo 'eval "$(_RFD_COMPLETE=source rfd)"' >> ~/.profile
 ```
 
 ### zsh
 
-
+```console
-```zsh
+$ echo 'eval "$(_RFD_COMPLETE=source_zsh rfd)"' >> ~/.zshrc
-echo 'eval "$(_RFD_COMPLETE=source_zsh rfd)"' >> ~/.zshrc
 ```

requirements.txt

@@ -1,4 +1,5 @@
-beautifulsoup4>=4.6.0
+beautifulsoup4<=4.8.2
 click>=7.0
-colorama>=0.3.9
+colorama>=0.4.3
-requests>=2.18.0
+requests>=2.22.0
+soupsieve<=2.0

requirements_dev.txt

@@ -1,5 +1,5 @@
-pre-commit==1.20.0
+pre-commit==1.21.0
 pylint
 pytest>=4.6.6
-rope==0.14.0
+rope==0.16.0
 tox-travis==0.12

rfd/VERSION

@@ -1 +1 @@
-0.3.3
+0.6.0

rfd/__init__.py

@@ -6,3 +6,5 @@ __title__ = "RFD CLI"
 __author__ = "Dave Gallant"
 __license__ = "Apache 2.0"
 __copyright__ = "(c) 2018 Dave Gallant"
+
+API_BASE_URL = "https://forums.redflagdeals.com"

rfd/api.py

@@ -6,11 +6,10 @@ except ImportError:
     JSONDecodeError = ValueError
 import logging
 import requests
-from .constants import API_BASE_URL
+from . import API_BASE_URL
-from .format import strip_html, is_valid_url
+from .posts import Post
-from .models import Post
 from .scores import calculate_score
-from .utils import is_int
+from .utils import is_int, strip_html, is_valid_url
 
 
 def extract_post_id(url):
@@ -34,28 +33,30 @@ def create_user_map(users):
     return m
 
 
-def get_threads(forum_id, limit, page=1):
+def get_threads(forum_id, pages):
     """Get threads from rfd api
 
     Arguments:
         forum_id {int} -- forum id
-        limit {[type]} -- limit number of threads returned
+        pages {int} -- the number of pages of threads to collect
 
     Returns:
         dict -- api response
     """
+    threads = []
     try:
-        response = requests.get(
+        for page in range(1, pages + 1):
-            "{}/api/topics?forum_id={}&per_page={}&page={}".format(
+            response = requests.get(
-                API_BASE_URL, forum_id, get_safe_per_page(limit), page
+                "{}/api/topics?forum_id={}&per_page=40&page={}".format(
+                    API_BASE_URL, forum_id, page
+                )
             )
-        )
+            if response.status_code != 200:
-        if response.status_code == 200:
+                raise Exception("When collecting threads, received a status code: %s" % response.status_code)
-            return response.json()
+            threads += response.json().get("topics")
-        logging.error("Unable to retrieve threads. %s", response.text)
     except JSONDecodeError as err:
-        logging.error("Unable to retrieve threads. %s", err)
+        logging.error("Unable to decode threads. %s", err)
-    return None
+    return threads
 
 
 def get_posts(post):

rfd/cli.py

@@ -2,17 +2,15 @@ from __future__ import unicode_literals
 
 
 import logging
-import os
 import sys
 import click
-from colorama import init, Fore, Style
+from colorama import init
 from .api import get_threads, get_posts
-from .search import search_threads
+from .threads import parse_threads, search_threads, sort_threads, generate_thread_output
-from .parsing import parse_threads
+from .posts import generate_posts_output
 from .__version__ import version as current_version
 
 init()
-print()
 
 logging.getLogger()
 logging.getLogger().setLevel(logging.INFO)
@@ -20,65 +18,46 @@ logging.getLogger().addHandler(logging.StreamHandler())
 
 
 def get_version():
-    return "rfd " + current_version
+    return "rfd v" + current_version
 
-
+def print_version(ctx, value):
-def get_terminal_width():
+    if not value or ctx.resilient_parsing:
-    _, columns = os.popen("stty size", "r").read().split()
+        return
-    return int(columns)
+    click.echo(get_version(), nl=False)
-
+    ctx.exit()
-
-def get_vote_color(score):
-    if score > 0:
-        return Fore.GREEN + " [+" + str(score) + "] "
-    if score < 0:
-        return Fore.RED + " [" + str(score) + "] "
-    return Fore.BLUE + " [" + str(score) + "] "
 
 
 @click.group(invoke_without_command=True)
-@click.option("--version/--no-version", default=False)
+@click.option(
+    "-v",
+    "--version",
+    is_flag=True,
+    callback=print_version,
+    expose_value=False,
+    is_eager=True,
+)
 @click.pass_context
-def cli(ctx, version):
+def cli(ctx):
     """CLI for https://forums.redflagdeals.com"""
-    if version:
+    if not ctx.invoked_subcommand:
-        click.echo(get_version())
-    elif not ctx.invoked_subcommand:
         click.echo(ctx.get_help())
 
 
-@cli.command("version")
+@cli.command(short_help="Display all posts in a thread.")
-def display_version():
-    click.echo(get_version())
-
-
-@cli.command(short_help="Displays posts in a specific thread.")
 @click.argument("post_id")
 def posts(post_id):
-    """Displays posts in a specific thread.
+    """Iterate all pages and display all posts in a thread.
 
     post_id can be a full url or post id only
 
     Example:
 
     \b
-    url: https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603
+    rfd posts https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603
-    post_id: 2173603
     """
 
     try:
-        click.echo("-" * get_terminal_width())
+        click.echo_via_pager(generate_posts_output(get_posts(post=post_id)))
-        for post in get_posts(post=post_id):
-            click.echo(
-                " -"
-                + get_vote_color(post.score)
-                + Fore.RESET
-                + post.body
-                + Fore.YELLOW
-                + " ({})".format(post.user)
-            )
-            click.echo(Style.RESET_ALL)
-            click.echo("-" * get_terminal_width())
     except ValueError:
         click.echo("Invalid post id.")
         sys.exit(1)
@@ -87,11 +66,12 @@ def posts(post_id):
         sys.exit(1)
 
 
-@cli.command(short_help="Displays threads in the specified forum.")
+@cli.command(short_help="Displays threads in the forum. Defaults to hot deals.")
-@click.option("--limit", default=10, help="Number of topics.")
 @click.option("--forum-id", default=9, help="The forum id number")
-def threads(limit, forum_id):
+@click.option("--pages", default=1, help="Number of pages to show. Defaults to 1.")
-    """Displays threads in the specified forum id. Defaults to 9.
+@click.option("--sort-by", default=None, help="Sort threads by")
+def threads(forum_id, pages, sort_by):
+    """Display threads in the specified forum id. Defaults to 9 (hot deals).
 
     Popular forum ids:
 
@@ -107,28 +87,19 @@ def threads(limit, forum_id):
     74 \t shopping discussion
     88 \t cell phones
     """
-    _threads = parse_threads(get_threads(forum_id, limit), limit)
+    _threads = sort_threads(parse_threads(get_threads(forum_id, pages)), sort_by=sort_by)
-    for count, thread in enumerate(_threads, 1):
+    click.echo_via_pager(generate_thread_output(_threads))
-        click.echo(
-            " "
-            + str(count)
-            + "."
-            + get_vote_color(thread.score)
-            + Fore.RESET
-            + "[%s] %s" % (thread.dealer_name, thread.title)
-        )
-        click.echo(Fore.BLUE + " {}".format(thread.url))
-        click.echo(Style.RESET_ALL)
 
 
-@cli.command(short_help="Displays threads in the specified forum.")
+@cli.command(short_help="Search deals based on a regular expression.")
-@click.option("--num-pages", default=5, help="Number of pages to search.")
+@click.option("--pages", default=5, help="Number of pages to search.")
 @click.option(
     "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)."
 )
-@click.argument("keyword")
+@click.option("--sort-by", default=None, help="Sort threads by")
-def search(num_pages, forum_id, keyword):
+@click.argument("regex")
-    """Searches for deals based on a keyword in the specified forum id.
+def search(pages, forum_id, sort_by, regex):
+    """Search deals based on regex.
 
     Popular forum ids:
 
@@ -145,18 +116,11 @@ def search(num_pages, forum_id, keyword):
     88 \t cell phones
     """
 
-    count = 0
+    matched_threads = []
-    for page in range(1, num_pages):
+
-        _threads = parse_threads(get_threads(forum_id, 100, page=page), limit=100)
+    _threads = parse_threads(get_threads(forum_id, pages=pages))
-        for thread in search_threads(threads=_threads, keyword=keyword):
+    for thread in search_threads(threads=_threads, regex=regex):
-            count += 1
+        matched_threads.append(thread)
-            click.echo(
+    click.echo_via_pager(
-                " "
+        generate_thread_output(sort_threads(matched_threads, sort_by=sort_by))
-                + str(count)
+    )
-                + "."
-                + get_vote_color(thread.score)
-                + Fore.RESET
-                + "[%s] %s" % (thread.dealer_name, thread.title)
-            )
-            click.echo(Fore.BLUE + " {}".format(thread.url))
-            click.echo(Style.RESET_ALL)

rfd/constants.py

@@ -1 +0,0 @@
-API_BASE_URL = "https://forums.redflagdeals.com"

rfd/format.py

@@ -1,16 +0,0 @@
-"""Formatting utils"""
-
-try:
-    from urllib.parse import urlparse  # python 3
-except ImportError:
-    from urlparse import urlparse  # python 2
-from bs4 import BeautifulSoup
-
-
-def strip_html(text):
-    return BeautifulSoup(text, "html.parser").get_text()
-
-
-def is_valid_url(url):
-    result = urlparse(url)
-    return all([result.scheme, result.netloc, result.path])

rfd/models.py

@@ -1,17 +0,0 @@
-# pylint: disable=old-style-class
-class Thread:
-    def __init__(self, title, dealer_name, score, url):
-        self.dealer_name = dealer_name
-        self.score = score
-        self.title = title
-        self.url = url
-
-    def __repr__(self):
-        return "Thread(%s)" % self.title
-
-
-class Post:
-    def __init__(self, body, score, user):
-        self.body = body
-        self.score = score
-        self.user = user

rfd/parsing.py

@@ -1,34 +0,0 @@
-from .constants import API_BASE_URL
-from .scores import calculate_score
-from .models import Thread
-
-
-def build_web_path(slug):
-    return "{}{}".format(API_BASE_URL, slug)
-
-
-def parse_threads(threads, limit):
-    """parse topics list api response into digestible list.
-
-    Arguments:
-        threads {dict} -- topics response from rfd api
-        limit {int} -- limit number of threads returned
-
-    Returns:
-        list(dict) -- digestible list of threads
-    """
-    parsed_threads = []
-    if threads is None:
-        return []
-    for count, topic in enumerate(threads.get("topics"), start=1):
-        if count > limit:
-            break
-        parsed_threads.append(
-            Thread(
-                title=topic.get("title"),
-                dealer_name=topic["offer"].get("dealer_name"),
-                score=calculate_score(topic),
-                url=build_web_path(topic.get("web_path")),
-            )
-        )
-    return parsed_threads

rfd/posts.py

@@ -0,0 +1,32 @@
+# pylint: disable=old-style-class
+import os
+from colorama import Fore, Style
+from .scores import get_vote_color
+
+class Post:
+    def __init__(self, body, score, user):
+        self.body = body
+        self.score = score
+        self.user = user
+
+def get_terminal_width():
+    _, columns = os.popen("stty size", "r").read().split()
+    return int(columns)
+
+def generate_posts_output(posts):
+    output = ""
+    output += ("-" * get_terminal_width())
+    for post in posts:
+        output += (
+            " -"
+            + get_vote_color(post.score)
+            + Fore.RESET
+            + post.body
+            + Fore.YELLOW
+            + " ({})".format(post.user)
+        )
+        output += (Style.RESET_ALL)
+        output += "\n"
+        output += ("-" * get_terminal_width())
+        output += "\n"
+    return output

rfd/scores.py

@@ -1,3 +1,6 @@
+from colorama import Fore
+
+
 def calculate_score(post):
     """Calculate either topic or post score. If votes cannot be retrieved, the score is 0.
 
@@ -16,3 +19,11 @@ def calculate_score(post):
         pass
 
     return score
+
+
+def get_vote_color(score):
+    if score > 0:
+        return Fore.GREEN + " [+" + str(score) + "] "
+    if score < 0:
+        return Fore.RED + " [" + str(score) + "] "
+    return Fore.BLUE + " [" + str(score) + "] "

rfd/search.py

@@ -1,13 +0,0 @@
-def search_threads(threads, keyword=None):
-    """Match deal title and dealer names with keyword specified."""
-
-    if keyword is None:
-        return
-
-    keyword = str(keyword)
-
-    for deal in threads:
-        if keyword.lower() in deal.title.lower() or (
-            deal.dealer_name and keyword.lower() in deal.dealer_name.lower()
-        ):
-            yield deal

rfd/threads.py

@@ -0,0 +1,99 @@
+import re
+from colorama import Fore, Style
+from . import API_BASE_URL
+from .scores import calculate_score, get_vote_color
+
+# pylint: disable=old-style-class
+class Thread:
+    def __init__(self, title, dealer_name, score, url, views):
+        self.dealer_name = dealer_name
+        self.score = score
+        self.title = title
+        self.url = url
+        self.views = views
+
+    def __repr__(self):
+        return "Thread(%s)" % self.title
+
+
+def build_web_path(slug):
+    return "{}{}".format(API_BASE_URL, slug)
+
+
+def get_dealer(topic):
+    dealer = None
+    if topic.get("offer"):
+        dealer = topic.get("offer").get("dealer_name")
+    return dealer
+
+
+def parse_threads(threads):
+    """Parse topics list api response into digestible list.
+
+    Arguments:
+        threads {dict} -- topics response from rfd api
+
+    Returns:
+        list(dict) -- digestible list of threads
+    """
+    parsed_threads = []
+    if threads is None:
+        return []
+    for topic in threads:
+        parsed_threads.append(
+            Thread(
+                title=topic.get("title"),
+                dealer_name=get_dealer(topic),
+                score=calculate_score(topic),
+                url=build_web_path(topic.get("web_path")),
+                views=topic.get("total_views"),
+            )
+        )
+    return parsed_threads
+
+
+def sort_threads(threads, sort_by):
+    """Sort threads by an attribute"""
+    if sort_by is None:
+        return threads
+    assert sort_by in ["views", "score", "title"]
+    threads = sorted(threads, key=lambda x: getattr(x, sort_by), reverse=True)
+    return threads
+
+
+def search_threads(threads, regex):
+    """Match deal title and dealer names with regex specified."""
+
+    regexp = re.compile(str(regex).lower())
+
+    for deal in threads:
+
+        if regexp.search(deal.title.lower()) or (
+            deal.dealer_name and regexp.search(deal.dealer_name.lower())
+        ):
+            yield deal
+
+
+def generate_thread_output(threads):
+    for count, thread in enumerate(threads, 1):
+        output = ""
+        dealer = thread.dealer_name
+        if dealer and dealer is not None:
+            dealer = "[" + dealer + "] "
+        else:
+            dealer = ""
+        output += (
+            " "
+            + str(count)
+            + "."
+            + get_vote_color(thread.score)
+            + Fore.RESET
+            + "%s%s" % (dealer, thread.title)
+            + Fore.LIGHTYELLOW_EX
+            + " (%d views)" % thread.views
+            + Fore.RESET
+        )
+        output += Fore.BLUE + " {}".format(thread.url)
+        output += Style.RESET_ALL
+        output += "\n\n"
+        yield output

rfd/utils.py

@@ -1,4 +1,18 @@
 """This module provides utility functions that are used within rfd"""
+try:
+    from urllib.parse import urlparse  # python 2
+except ImportError:
+    from urlparse import urlparse  # python 1
+from bs4 import BeautifulSoup
+
+
+def strip_html(text):
+    return BeautifulSoup(text, "html.parser").get_text()
+
+
+def is_valid_url(url):
+    result = urlparse(url)
+    return all([result.scheme, result.netloc, result.path])
 
 
 def is_int(number):

tests/integration/test_cli.py

@@ -0,0 +1,25 @@
+from subprocess import Popen, PIPE
+import pytest
+
+
+def run_cli(args):
+    cmd = ["python", "-m", "rfd"] + args.split()
+    p = Popen(cmd, stdout=PIPE)
+    stdout, _ = p.communicate()
+    assert p.returncode == 0
+    return stdout
+
+
+def test_version():
+    stdout = run_cli("--version")
+    assert b"rfd v" in stdout
+
+
+@pytest.mark.parametrize("args", ["", "--sort-by score"])
+def test_threads(args):
+    run_cli("threads " + args)
+
+
+@pytest.mark.parametrize("args", ["'pizza'", "'(coffee|starbucks)'"])
+def test_search(args):
+    run_cli("search " + args)

tests/test_api.py

@@ -1,5 +1,5 @@
 from rfd.api import extract_post_id
-from rfd.parsing import build_web_path, parse_threads
+from rfd.threads import build_web_path, parse_threads
 
 
 def test_build_web_path():
@@ -19,11 +19,10 @@ def test_extract_post_id():
 
 def test_parse_threads(threads_api_response):
 
-    limit = 10
+    threads = parse_threads(threads_api_response.get("topics"))
-    threads = parse_threads(threads_api_response, limit)
+    assert len(threads) == 10
-    assert len(threads) == limit
 
 
 def test_parse_threads_empty():
 
-    assert parse_threads(None, 10) == []
+    assert parse_threads(None) == []