Make a display thread function for re-use (#74)

* Add python38 to travis

* Add python39-dev

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1 @@
-
-**What this PR does / why we need it:**
--
-
-**Which issue(s) this PR fixes:**
--
-
-**Additional Notes:**
--
+###### Motivation for this change

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,27 @@
+name: "Code scanning - action"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 4 * * 1'
+
+jobs:
+  CodeQL-Build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.gitignore

@@ -88,3 +88,5 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
+
+tmp/

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-dist: xenial
+dist: bionic
 sudo: false
 cache: false
 stages:
@@ -18,15 +18,14 @@ script:
 jobs:
   include:
     - python: "2.7"
-
     - python: "3.5"
-
     - python: "3.6"
-
     - python: "3.7"
+    - python: "3.8"
+    - python: "3.9-dev"
 
     - stage: deploy
-      python: "3.7"
+      python: "3.8"
       deploy:
         on:
           repo: davegallant/rfd

Makefile

@@ -1,35 +1,57 @@
 
-SRC:=rfd
+SRC := rfd
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
 
-.PHONY: build
+
+ifeq ($(origin .RECIPEPREFIX), undefined)
+  $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
+endif
+.RECIPEPREFIX = >
+
+## build: Build a tar.gz of the python package
 build:
-	rm -rf dist/
-	python setup.py sdist
-
-.PHONY: push_test
-push_test:
-	twine upload -r testpypi dist/*.tar.gz
-
-.PHONY: push_prod
-push_prod:
-	twine upload dist/*.tar.gz
+> rm -rf dist/
+> python setup.py sdist
+.PHONY: build
 
+## precommit: Run all pre-commit hooks
+precommit:
+> pre-commit run \
+ --all-files \
+ --show-diff-on-failure
 .PHONY: precommit
-precommit: ## Run pre-commit
-	pre-commit run \
-	--all-files \
-	--show-diff-on-failure
 
-.PHONY: lint
+## lint: Run static analysis on the code
 lint:
-	pylint $(SRC)
+> pylint $(SRC)
+.PHONY: lint
 
+## test: Run all unit tests
+test: tmp/.tests-passed.sentinel
 .PHONY: test
-test:
-	pytest -v
 
-.PHONY: pr
+# Tests - re-ran if any file under src has been changed since tmp/.tests-passed.sentinel was last touched
+tmp/.tests-passed.sentinel: $(shell find ${SRC} -type f)
+> mkdir -p $(@D)
+> pytest -v
+> touch $@
+
+## pr: Run pre-commit, lint and test
 pr: precommit lint test
+.PHONY: pr
 
-.PHONY: ci
 ci: lint test
+.PHONY: ci
+
+## help: Print this help message
+help:
+> @echo "Usage:"
+> @echo
+> @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /' | sort
+> @echo
+.PHONY: help

README.md

@@ -9,7 +9,8 @@ Hot deals on the command line.
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/davegallant/rfd.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/davegallant/rfd/context:python)
 
 
-![image](https://user-images.githubusercontent.com/4519234/71054408-e18c6a00-211f-11ea-89bc-3f990a4909de.png)
+![screenshot](https://user-images.githubusercontent.com/4519234/85969861-e10a4100-b996-11ea-9a31-6203322c60ee.png)
+
 
 ## Install
 
@@ -17,19 +18,38 @@ Hot deals on the command line.
 pip install rfd
 ```
 
-## Use
+## Usage
 
-### view threads
-```bash
-rfd threads [--forum-id 9] [--limit 10]
+```shell
+Usage: rfd [OPTIONS] COMMAND [ARGS]...
+
+  CLI for https://forums.redflagdeals.com
+
+Options:
+  -v, --version
+  --help         Show this message and exit.
+
+Commands:
+  posts    Display all posts in a thread.
+  search   Search deals based on a regular expression.
+  threads  Displays threads in the forum. Defaults to hot deals.
 ```
 
-### search
-```bash
-rfd search pizza [--num-pages 100]
+## Examples
+
+### view hot deals
+```shell
+rfd threads
 ```
 
-## Support Tab Completion
+### search for pizza
+```shell
+rfd search 'pizza'
+```
+
+## Tab Completion
+
+To enable:
 
 ### bash
 

requirements.txt

@@ -1,4 +1,5 @@
-beautifulsoup4>=4.8.1
+beautifulsoup4<=4.8.2
 click>=7.0
 colorama>=0.4.3
 requests>=2.22.0
+soupsieve<=2.0

requirements_dev.txt

@@ -1,5 +1,5 @@
-pre-commit==1.20.0
+pre-commit==1.21.0
 pylint
 pytest>=4.6.6
-rope==0.14.0
+rope==0.16.0
 tox-travis==0.12

rfd/VERSION

@@ -1 +1 @@
-0.3.4
+0.3.6

rfd/cli.py

@@ -36,34 +36,61 @@ def get_vote_color(score):
     return Fore.BLUE + " [" + str(score) + "] "
 
 
+def print_version(ctx, value):
+    if not value or ctx.resilient_parsing:
+        return
+    click.echo(get_version())
+    ctx.exit()
+
+
+def display_thread(click, thread, count):  # pylint: disable=redefined-outer-name
+    dealer = thread.dealer_name
+    if dealer and dealer is not None:
+        dealer = "[" + dealer + "] "
+    else:
+        dealer = ""
+    click.echo(
+        " "
+        + str(count)
+        + "."
+        + get_vote_color(thread.score)
+        + Fore.RESET
+        + "%s%s" % (dealer, thread.title)
+        + Fore.LIGHTYELLOW_EX
+        + " (%d views)" % thread.total_views
+        + Fore.RESET
+    )
+    click.echo(Fore.BLUE + " {}".format(thread.url))
+    click.echo(Style.RESET_ALL)
+
+
 @click.group(invoke_without_command=True)
-@click.option("--version/--no-version", default=False)
+@click.option(
+    "-v",
+    "--version",
+    is_flag=True,
+    callback=print_version,
+    expose_value=False,
+    is_eager=True,
+)
 @click.pass_context
-def cli(ctx, version):
+def cli(ctx):
     """CLI for https://forums.redflagdeals.com"""
-    if version:
-        click.echo(get_version())
-    elif not ctx.invoked_subcommand:
+    if not ctx.invoked_subcommand:
         click.echo(ctx.get_help())
 
 
-@cli.command("version")
-def display_version():
-    click.echo(get_version())
-
-
-@cli.command(short_help="Displays posts in a specific thread.")
+@cli.command(short_help="Display all posts in a thread.")
 @click.argument("post_id")
 def posts(post_id):
-    """Displays posts in a specific thread.
+    """Iterate all pages and display all posts in a thread.
 
     post_id can be a full url or post id only
 
     Example:
 
     \b
-    url: https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603
-    post_id: 2173603
+    rfd posts https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603
     """
 
     try:
@@ -87,11 +114,11 @@ def posts(post_id):
         sys.exit(1)
 
 
-@cli.command(short_help="Displays threads in the specified forum.")
+@cli.command(short_help="Displays threads in the forum. Defaults to hot deals.")
 @click.option("--limit", default=10, help="Number of topics.")
 @click.option("--forum-id", default=9, help="The forum id number")
 def threads(limit, forum_id):
-    """Displays threads in the specified forum id. Defaults to 9.
+    """Display threads in the specified forum id. Defaults to 9 (hot deals).
 
     Popular forum ids:
 
@@ -109,29 +136,17 @@ def threads(limit, forum_id):
     """
     _threads = parse_threads(get_threads(forum_id, limit), limit)
     for count, thread in enumerate(_threads, 1):
-        click.echo(
-            " "
-            + str(count)
-            + "."
-            + get_vote_color(thread.score)
-            + Fore.RESET
-            + "[%s] %s" % (thread.dealer_name, thread.title)
-            + Fore.LIGHTYELLOW_EX
-            + " (%d views)" % thread.total_views
-            + Fore.RESET
-        )
-        click.echo(Fore.BLUE + " {}".format(thread.url))
-        click.echo(Style.RESET_ALL)
+        display_thread(click, thread, count)
 
 
-@cli.command(short_help="Displays threads in the specified forum.")
+@cli.command(short_help="Search deals based on a regular expression.")
 @click.option("--num-pages", default=5, help="Number of pages to search.")
 @click.option(
     "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)."
 )
-@click.argument("keyword")
-def search(num_pages, forum_id, keyword):
-    """Searches for deals based on a keyword in the specified forum id.
+@click.argument("regex")
+def search(num_pages, forum_id, regex):
+    """Search deals based on regex.
 
     Popular forum ids:
 
@@ -151,15 +166,6 @@ def search(num_pages, forum_id, keyword):
     count = 0
     for page in range(1, num_pages):
         _threads = parse_threads(get_threads(forum_id, 100, page=page), limit=100)
-        for thread in search_threads(threads=_threads, keyword=keyword):
+        for thread in search_threads(threads=_threads, regex=regex):
             count += 1
-            click.echo(
-                " "
-                + str(count)
-                + "."
-                + get_vote_color(thread.score)
-                + Fore.RESET
-                + "[%s] %s" % (thread.dealer_name, thread.title)
-            )
-            click.echo(Fore.BLUE + " {}".format(thread.url))
-            click.echo(Style.RESET_ALL)
+            display_thread(click, thread, count)

rfd/parsing.py

@@ -7,6 +7,13 @@ def build_web_path(slug):
     return "{}{}".format(API_BASE_URL, slug)
 
 
+def get_dealer(topic):
+    dealer = None
+    if topic.get("offer"):
+        dealer = topic.get("offer").get("dealer_name")
+    return dealer
+
+
 def parse_threads(threads, limit):
     """parse topics list api response into digestible list.
 
@@ -26,7 +33,7 @@ def parse_threads(threads, limit):
         parsed_threads.append(
             Thread(
                 title=topic.get("title"),
-                dealer_name=topic["offer"].get("dealer_name"),
+                dealer_name=get_dealer(topic),
                 score=calculate_score(topic),
                 url=build_web_path(topic.get("web_path")),
                 total_views=topic.get("total_views"),

rfd/search.py

@@ -1,13 +1,14 @@
-def search_threads(threads, keyword=None):
-    """Match deal title and dealer names with keyword specified."""
+import re
 
-    if keyword is None:
-        return
 
-    keyword = str(keyword)
+def search_threads(threads, regex):
+    """Match deal title and dealer names with regex specified."""
+
+    regexp = re.compile(str(regex).lower())
 
     for deal in threads:
-        if keyword.lower() in deal.title.lower() or (
-            deal.dealer_name and keyword.lower() in deal.dealer_name.lower()
+
+        if regexp.search(deal.title.lower()) or (
+            deal.dealer_name and regexp.search(deal.dealer_name.lower())
         ):
             yield deal