From aed23b2b46ae0eab2331c2eb798d4a4dd3f99948 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Sun, 21 Aug 2022 14:27:27 -0400 Subject: [PATCH 01/15] Robustify Makefile Fixes: - Updates packer version - Updates packer-plugin-arm-image name - Downloads packer plugin via packer CLI instead of building from source - Supports image builds on more machines by downloading the correct packer release for the current build machine's CPU type - Dependencies added to 'image' target to ensure that packer is installed and that the image is only rebuilt when the input files are updated - Prevents Ansible from changing build machine's hostname Signed-off-by: llamasoft --- Makefile | 82 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 2c485719d..cb6f74448 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,30 @@ -PACKER_VERSION=1.7.2 -PWN_HOSTNAME=pwnagotchi -PWN_VERSION=master +PACKER_VERSION := 1.8.3 +PWN_HOSTNAME := pwnagotchi +PWN_VERSION := master +PWN_RELEASE := pwnagotchi-raspbian-lite-$(PWN_VERSION) + +MACHINE_TYPE := $(shell uname -m) +ifneq (,$(filter x86_64,$(MACHINE_TYPE))) +GOARCH := amd64 +else ifneq (,$(filter i686,$(MACHINE_TYPE))) +GOARCH := 386 +else ifneq (,$(filter arm64% aarch64%,$(MACHINE_TYPE))) +GOARCH := arm64 +else ifneq (,$(filter arm%,$(MACHINE_TYPE))) +GOARCH := arm +else +GOARCH := amd64 +$(warning Unable to detect CPU arch from machine type $(MACHINE_TYPE), assuming $(GOARCH)) +endif + +# The Ansible part of the build can inadvertently change the active hostname of +# the build machine while updating the permanent hostname of the build image. +# If the unshare command is available, use it to create a separate namespace +# so hostname changes won't affect the build machine. +UNSHARE := $(shell command -v unshare) +ifneq (,$(UNSHARE)) +UNSHARE := $(UNSHARE) --uts +endif all: clean install image @@ -8,23 +32,39 @@ langs: @for lang in pwnagotchi/locale/*/; do\ echo "compiling language: $$lang ..."; \ ./scripts/language.sh compile $$(basename $$lang); \ - done - -install: - curl https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_linux_amd64.zip -o /tmp/packer.zip - unzip /tmp/packer.zip -d /tmp - sudo mv /tmp/packer /usr/bin/packer - git clone https://github.com/solo-io/packer-builder-arm-image /tmp/packer-builder-arm-image - cd /tmp/packer-builder-arm-image && go get -d ./... && go build - sudo cp /tmp/packer-builder-arm-image/packer-builder-arm-image /usr/bin - -image: - cd builder && sudo /usr/bin/packer build -var "pwn_hostname=$(PWN_HOSTNAME)" -var "pwn_version=$(PWN_VERSION)" pwnagotchi.json - sudo mv builder/output-pwnagotchi/image pwnagotchi-raspbian-lite-$(PWN_VERSION).img - sudo sha256sum pwnagotchi-raspbian-lite-$(PWN_VERSION).img > pwnagotchi-raspbian-lite-$(PWN_VERSION).sha256 - sudo zip pwnagotchi-raspbian-lite-$(PWN_VERSION).zip pwnagotchi-raspbian-lite-$(PWN_VERSION).sha256 pwnagotchi-raspbian-lite-$(PWN_VERSION).img + done + +PACKER := /tmp/pwnagotchi/packer +PACKER_URL := https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_linux_$(GOARCH).zip +$(PACKER): + mkdir -p $(@D) + curl -L "$(PACKER_URL)" -o $(PACKER).zip + unzip $(PACKER).zip -d $(@D) + rm $(PACKER).zip + chmod +x $@ + +# Building the image requires packer, but don't rebuild the image just because packer updated. +$(PWN_RELEASE).img: | $(PACKER) + +# If the packer or ansible files are updated, rebuild the image. +$(PWN_RELEASE).img: builder/pwnagotchi.json builder/pwnagotchi.yml $(shell find builder/data -type f) + sudo $(PACKER) plugins install github.com/solo-io/arm-image + cd builder && sudo $(UNSHARE) $(PACKER) build -var "pwn_hostname=$(PWN_HOSTNAME)" -var "pwn_version=$(PWN_VERSION)" pwnagotchi.json + sudo chown -R $$USER:$$USER builder/output-pwnagotchi + mv builder/output-pwnagotchi/image $@ + +# If any of these files are updated, rebuild the checksums. +$(PWN_RELEASE).sha256: $(PWN_RELEASE).img + sha256sum $^ > $@ + +# If any of the input files are updated, rebuild the archive. +$(PWN_RELEASE).zip: $(PWN_RELEASE).img $(PWN_RELEASE).sha256 + zip $(PWN_RELEASE).zip $^ + +.PHONY: image +image: $(PWN_RELEASE).zip clean: - rm -rf /tmp/packer-builder-arm-image - rm -f pwnagotchi-raspbian-lite-*.zip pwnagotchi-raspbian-lite-*.img pwnagotchi-raspbian-lite-*.sha256 - rm -rf builder/output-pwnagotchi builder/packer_cache + rm -f $(PACKER) + rm -f $(PWN_RELEASE).* + sudo rm -rf builder/output-pwnagotchi builder/packer_cache From a7f03e39ace9c00e20895195376c3ac21fb0aed1 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Sun, 21 Aug 2022 14:31:12 -0400 Subject: [PATCH 02/15] Update apt-get command to support oldstable Raspbian has been deprecated in favor of RaspiOS. As a consequence, the Raspbian repo has changed from 'stable' to 'oldstable' which causes the apt-get update command to fail. This is the minimal change required to get the outdated image to build correctly. Signed-off-by: llamasoft --- builder/pwnagotchi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/pwnagotchi.json b/builder/pwnagotchi.json index 8f1a09588..e49c6678d 100644 --- a/builder/pwnagotchi.json +++ b/builder/pwnagotchi.json @@ -14,7 +14,7 @@ "inline": [ "sed -i 's/^\\([^#]\\)/#\\1/g' /etc/ld.so.preload", "dpkg-architecture", - "apt-get -y update", + "apt-get -y --allow-releaseinfo-change update", "apt-get install -y ansible" ] }, From e220ccece9b131675a3f61690f6bda5759781f81 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Wed, 31 Aug 2022 21:33:05 +0000 Subject: [PATCH 03/15] Update re4son-kernel repo key The PGP signature file for the re4son-kernel repo expired on August 29, 2022. The re-signed PGP key is available from the MIT PGP keyserver. Signed-off-by: llamasoft --- builder/pwnagotchi.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/pwnagotchi.yml b/builder/pwnagotchi.yml index 0ee66708e..f19e92733 100644 --- a/builder/pwnagotchi.yml +++ b/builder/pwnagotchi.yml @@ -130,7 +130,8 @@ - name: Add re4son-kernel repo key apt_key: - url: https://re4son-kernel.com/keys/http/archive-key.asc + id: "11764EE8AC24832F" + keyserver: "keyserver.ubuntu.com" state: present - name: Add re4son-kernel repository From fb07032b9c54f1ed012d9afc16622201ab6e8e93 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Thu, 1 Sep 2022 19:59:43 -0400 Subject: [PATCH 04/15] Update requirements to fix broken imports flask, Jinja2, Werkzeug, and itsdangerous are all from Pallets Projects who seemingly refuse to use PEP-440's ~= "compatible release" clause. Instead, all of their project dependencies are ">=" which eventually causes things to break. Their canned response is to recommend always staying on the latest version and to use something like pip-tools to pin dependencies. https://github.com/pallets/markupsafe/issues/282#issuecomment-1043899655 Signed-off-by: llamasoft --- requirements.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/requirements.txt b/requirements.txt index df47e7fb0..2730513c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,24 @@ dbus-python==1.2.12 toml==0.10.0 python-dateutil==2.8.1 websockets==8.1 + +# flask==1.0.2 requires: +# Jinja2>=2.10 +# Werkzeug>=0.14 +# itsdangerous>=0.24 + +# Jinja2 v2.1.0 requires MarkupSafe>=0.23 for `soft_unicode` +# but the function was removed in MarkupSafe v2.1.0. +# Jinja2 v3.0.0 fixed this issue but flask doesn't support it until v2.0. +MarkupSafe<2.1.0 + +# Werkzeug v2.2.0 adds a requirement of MarkupSafe>=2.1.1 +# which triggers the `soft_unicode` issue in Jinja2. +# Werkzeug v2.1.0 removes `safe_str_cmp` which breaks flask-wtf. +# flask-wtf fixes this in v0.15. +Werkzeug<2.1.0 + +# flask requires itsdangerous>=0.24 for its `json` module +# but the module was removed in v2.1.0. +# flask fixed this issue in v2.X. +itsdangerous<2.1.0 \ No newline at end of file From 13769f664d607b67fe60c42d511bd53e26e06827 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Sun, 21 Aug 2022 14:59:18 -0400 Subject: [PATCH 05/15] Bump RasPiOS to latest buster release Raspbian has been renamed to Raspberry Pi OS. RasPiOS on buster is considered legacy but images are still released for it. A few packages have to be uninstalled (from the main repo) and reinstalled (from the re4son-kernel repo) to fix dependency issues. Signed-off-by: llamasoft --- Makefile | 2 +- builder/pwnagotchi.json | 4 ++-- builder/pwnagotchi.yml | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index cb6f74448..4bac72cc1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PACKER_VERSION := 1.8.3 PWN_HOSTNAME := pwnagotchi PWN_VERSION := master -PWN_RELEASE := pwnagotchi-raspbian-lite-$(PWN_VERSION) +PWN_RELEASE := pwnagotchi-raspios-lite-$(PWN_VERSION) MACHINE_TYPE := $(shell uname -m) ifneq (,$(filter x86_64,$(MACHINE_TYPE))) diff --git a/builder/pwnagotchi.json b/builder/pwnagotchi.json index e49c6678d..b9d9decd4 100644 --- a/builder/pwnagotchi.json +++ b/builder/pwnagotchi.json @@ -3,8 +3,8 @@ { "name": "pwnagotchi", "type": "arm-image", - "iso_url": "https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2019-07-12/2019-07-10-raspbian-buster-lite.zip", - "iso_checksum": "9e5cf24ce483bb96e7736ea75ca422e3560e7b455eee63dd28f66fa1825db70e", + "iso_url": "https://downloads.raspberrypi.org/raspios_oldstable_lite_armhf/images/raspios_oldstable_lite_armhf-2022-04-07/2022-04-04-raspios-buster-armhf-lite.img.xz", + "iso_checksum": "42fd907a0da36b8a8ce9db9cd1cb77746b6a10c4b77f8d0ae0b8065a3b358a37", "last_partition_extra_size": 3221225472 } ], diff --git a/builder/pwnagotchi.yml b/builder/pwnagotchi.yml index f19e92733..00cb6bb48 100644 --- a/builder/pwnagotchi.yml +++ b/builder/pwnagotchi.yml @@ -53,6 +53,12 @@ - triggerhappy - wpa_supplicant - nfs-common + # Remove libraspberrypi so we can reinstall them from the kali-pi repo instead. + # This prevents kalipi-bootloader conflicts with raspberrypi-bootloader. + - libraspberrypi0 + - libraspberrypi-dev + - libraspberrypi-doc + - libraspberrypi-bin install: - rsync - vim From e4ad0b960d970110052b0d1a8ed6176c22dcfe3c Mon Sep 17 00:00:00 2001 From: llamasoft Date: Sat, 3 Sep 2022 22:56:24 +0000 Subject: [PATCH 06/15] Update qemu_args to use correct CPU The default ARM CPU emulated by qemu is armv7l. While this is fine for most packages, it leads to the wrong opencv and tensorflow being installed. By emulating the correct CPU, we're guaranteed to end up with the correct binaries. Signed-off-by: llamasoft --- builder/pwnagotchi.json | 3 ++- builder/pwnagotchi.yml | 12 ------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/builder/pwnagotchi.json b/builder/pwnagotchi.json index b9d9decd4..87c0a8e3b 100644 --- a/builder/pwnagotchi.json +++ b/builder/pwnagotchi.json @@ -5,7 +5,8 @@ "type": "arm-image", "iso_url": "https://downloads.raspberrypi.org/raspios_oldstable_lite_armhf/images/raspios_oldstable_lite_armhf-2022-04-07/2022-04-04-raspios-buster-armhf-lite.img.xz", "iso_checksum": "42fd907a0da36b8a8ce9db9cd1cb77746b6a10c4b77f8d0ae0b8065a3b358a37", - "last_partition_extra_size": 3221225472 + "target_image_size": 6442450944, + "qemu_args": ["-cpu", "arm1176"] } ], "provisioners": [ diff --git a/builder/pwnagotchi.yml b/builder/pwnagotchi.yml index 00cb6bb48..51079803d 100644 --- a/builder/pwnagotchi.yml +++ b/builder/pwnagotchi.yml @@ -259,18 +259,6 @@ chdir: /usr/local/src/pwnagotchi when: (pwnagotchigit.changed) or (pip_packages['pwnagotchi'] is undefined) or (pip_packages['pwnagotchi'] != pwnagotchi_version) - - name: install opencv-python - pip: - name: "https://www.piwheels.org/simple/opencv-python/opencv_python-3.4.3.18-cp37-cp37m-linux_armv6l.whl" - extra_args: "--no-deps --no-cache-dir --platform=linux_armv6l --only-binary=:all: --target={{ pip_target.stdout }}" - when: (pip_packages['opencv-python'] is undefined) or (pip_packages['opencv-python'] != '3.4.3.18') - - - name: install tensorflow - pip: - name: "https://www.piwheels.org/simple/tensorflow/tensorflow-1.13.1-cp37-none-linux_armv6l.whl" - extra_args: "--no-deps --no-cache-dir --platform=linux_armv6l --only-binary=:all: --target={{ pip_target.stdout }}" - when: (pip_packages['tensorflow'] is undefined) or (pip_packages['tensorflow'] != '1.13.1') - - name: install pwnagotchi wheel and dependencies pip: name: "{{ lookup('fileglob', '/usr/local/src/pwnagotchi/dist/pwnagotchi*.whl') }}" From 889ddabaa09fdfb3c819b11b9c5887d0221c8231 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Thu, 8 Sep 2022 00:20:42 +0000 Subject: [PATCH 07/15] Fix broken monitor interface If the monitor interface is created before the wlan0 interface is up, the resulting mon0 interface will be in a permanently broken state. Since the "ifup@wlan0.service" is disabled and getting timing guarantees from systemd is a challenge, we manually up the interface. Signed-off-by: llamasoft --- builder/data/usr/bin/pwnlib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/data/usr/bin/pwnlib b/builder/data/usr/bin/pwnlib index 0e45d2598..a314c7261 100755 --- a/builder/data/usr/bin/pwnlib +++ b/builder/data/usr/bin/pwnlib @@ -33,7 +33,7 @@ reload_brcm() { # starts mon0 start_monitor_interface() { - iw phy "$(iw phy | head -1 | cut -d" " -f2)" interface add mon0 type monitor && ifconfig mon0 up + ifconfig wlan0 up && iw phy "$(iw phy | head -1 | cut -d" " -f2)" interface add mon0 type monitor && ifconfig mon0 up } # stops mon0 From ea827a42e1ef0c296e764b8d5a98192997fe30fe Mon Sep 17 00:00:00 2001 From: llamasoft Date: Thu, 8 Sep 2022 04:43:23 +0000 Subject: [PATCH 08/15] Fix broken wlan0 interface config An interface in static mode is required to have an address defined, otherwise ifup will throw an error and fail to bring the interface up. The Debian wiki provided the pre-up/post-down solution: https://wiki.debian.org/NetworkConfiguration#Bringing_up_an_interface_without_an_IP_address Signed-off-by: llamasoft --- builder/data/etc/network/interfaces.d/wlan0-cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/data/etc/network/interfaces.d/wlan0-cfg b/builder/data/etc/network/interfaces.d/wlan0-cfg index f54256948..55d6efe90 100644 --- a/builder/data/etc/network/interfaces.d/wlan0-cfg +++ b/builder/data/etc/network/interfaces.d/wlan0-cfg @@ -1,2 +1,4 @@ allow-hotplug wlan0 -iface wlan0 inet static \ No newline at end of file +iface wlan0 inet manual + pre-up ifconfig $IFACE up + post-down ifconfig $IFACE down \ No newline at end of file From 3af03cbe1ab8451fffb3c42a98e4395deea52fb9 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Mon, 5 Sep 2022 21:07:46 +0000 Subject: [PATCH 09/15] Move system file install into setup.py The installation of system files has been moved into the 'install' command instead of running unconditionally. This makes it possible to create a source code distribution of that doesn't immediately clobber the build machine's system files. This will also allow the image build to be simplified as it will only need to copy the sdist archive instead of multiple directories worth of files. Signed-off-by: llamasoft --- MANIFEST.in | 3 ++- setup.py | 51 ++++++++++++++++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 8bf5cfb21..4c36686a5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,10 @@ exclude *.pyc .DS_Store .gitignore MANIFEST.in +include requirements.txt include setup.py -include distribute_setup.py include README.md include LICENSE recursive-include bin * +recursive-include builder/data * recursive-include pwnagotchi *.py recursive-include pwnagotchi *.yml recursive-include pwnagotchi *.* diff --git a/setup.py b/setup.py index 34df29774..7027f424e 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,31 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- from setuptools import setup, find_packages -from distutils.util import strtobool -import os +from setuptools.command.install import install import glob -import shutil +import logging +import os import re +import shutil +import warnings +log = logging.getLogger(__name__) def install_file(source_filename, dest_filename): # do not overwrite network configuration if it exists already # https://github.com/evilsocket/pwnagotchi/issues/483 if dest_filename.startswith('/etc/network/interfaces.d/') and os.path.exists(dest_filename): - print("%s exists, skipping ..." % dest_filename) + log.info(f"{dest_filename} exists, skipping ...") return - print("installing %s to %s ..." % (source_filename, dest_filename)) - try: - dest_folder = os.path.dirname(dest_filename) - if not os.path.isdir(dest_folder): - os.makedirs(dest_folder) + log.info(f"installing {source_filename} to {dest_filename} ...") + dest_folder = os.path.dirname(dest_filename) + if not os.path.isdir(dest_folder): + os.makedirs(dest_folder) - shutil.copyfile(source_filename, dest_filename) - except Exception as e: - print("error installing %s: %s" % (source_filename, e)) + shutil.copyfile(source_filename, dest_filename) + if dest_filename.startswith("/usr/bin/"): + os.chmod(dest_filename, 0o755) def install_system_files(): @@ -35,15 +37,27 @@ def install_system_files(): dest_filename = source_filename.replace(data_path, '') install_file(source_filename, dest_filename) + +def restart_services(): # reload systemd units os.system("systemctl daemon-reload") - -def installer(): - install_system_files() # for people updating https://github.com/evilsocket/pwnagotchi/pull/551/files os.system("systemctl enable fstrim.timer") + +class CustomInstall(install): + def run(self): + super().run() + if os.geteuid() != 0: + warnings.warn( + "Not running as root, can't install pwnagotchi system files!" + ) + return + install_system_files() + restart_services() + + def version(version_file): with open(version_file, 'rt') as vf: version_file_content = vf.read() @@ -54,10 +68,6 @@ def version(version_file): return None - -if strtobool(os.environ.get("PWNAGOTCHI_ENABLE_INSTALLER", "1")): - installer() - with open('requirements.txt') as fp: required = [line.strip() for line in fp if line.strip() != ""] @@ -72,6 +82,9 @@ def version(version_file): url='https://pwnagotchi.ai/', license='GPL', install_requires=required, + cmdclass={ + "install": CustomInstall, + }, scripts=['bin/pwnagotchi'], package_data={'pwnagotchi': ['defaults.yml', 'pwnagotchi/defaults.yml', 'locale/*/LC_MESSAGES/*.mo']}, include_package_data=True, From fd136ebc70d73df6c0e46f637d2d8513bc08e1e0 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Mon, 5 Sep 2022 21:48:04 +0000 Subject: [PATCH 10/15] Copy requirements.txt to requirements.in The idea here is that the requirements.in file should be the loosest allowed requirements for the project itself, then it should be pip-compile'd into a concrete requirements.txt that has known working values. This prevents unnecessarily pinning packages to specific releases which makes it easier to keep package versions up-to-date going forward. Signed-off-by: llamasoft --- requirements.in | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 requirements.in diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..2730513c8 --- /dev/null +++ b/requirements.in @@ -0,0 +1,45 @@ +pycryptodome==3.9.4 +requests==2.21.0 +PyYAML==5.3.1 +scapy==2.4.3 +gym==0.14.0 +scipy==1.3.1 +stable-baselines==2.7.0 +tensorflow==1.13.1 +tensorflow-estimator==1.14.0 +tweepy==3.7.0 +file-read-backwards==2.0.0 +numpy==1.20.2 +inky==1.2.0 +smbus2==0.3.0 +Pillow==5.4.1 +spidev==3.4 +gast==0.2.2 +flask==1.0.2 +flask-cors==3.0.7 +flask-wtf==0.14.3 +dbus-python==1.2.12 +toml==0.10.0 +python-dateutil==2.8.1 +websockets==8.1 + +# flask==1.0.2 requires: +# Jinja2>=2.10 +# Werkzeug>=0.14 +# itsdangerous>=0.24 + +# Jinja2 v2.1.0 requires MarkupSafe>=0.23 for `soft_unicode` +# but the function was removed in MarkupSafe v2.1.0. +# Jinja2 v3.0.0 fixed this issue but flask doesn't support it until v2.0. +MarkupSafe<2.1.0 + +# Werkzeug v2.2.0 adds a requirement of MarkupSafe>=2.1.1 +# which triggers the `soft_unicode` issue in Jinja2. +# Werkzeug v2.1.0 removes `safe_str_cmp` which breaks flask-wtf. +# flask-wtf fixes this in v0.15. +Werkzeug<2.1.0 + +# flask requires itsdangerous>=0.24 for its `json` module +# but the module was removed in v2.1.0. +# flask fixed this issue in v2.X. +itsdangerous<2.1.0 \ No newline at end of file From a65d89fd62947e755e811cee8f9548e40c15ed16 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Tue, 6 Sep 2022 01:27:34 +0000 Subject: [PATCH 11/15] Document and update requirements.in Firstly, all requirements were sorted. Then, any dependencies not directly used by pwnagotchi were removed (e.g. gast isn't used directly, but it will eventually be installed as an indirect dependency of tensorflow). Next, every single dependency was researched and documented to determine how it's used in addition to what versions it can be safely upgraded to. Most dependencies have been updated to use the PEP-440 "compatible release" feature. Lastly, the --extra-index and --prefer-binary options have been added. As a result of some unfortunate publication issues (e.g. grpcio v1.46.X), some non-yanked libraries simply cannot be built from source. The --prefer-binary option allows pip to prefer an older library version if it has a binary wheel available. This also results in faster builds as fewer requirements actually need to be compiled from source. Signed-off-by: llamasoft --- requirements.in | 128 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 43 deletions(-) diff --git a/requirements.in b/requirements.in index 2730513c8..b10b6be25 100644 --- a/requirements.in +++ b/requirements.in @@ -1,45 +1,87 @@ -pycryptodome==3.9.4 -requests==2.21.0 -PyYAML==5.3.1 -scapy==2.4.3 -gym==0.14.0 -scipy==1.3.1 -stable-baselines==2.7.0 -tensorflow==1.13.1 -tensorflow-estimator==1.14.0 -tweepy==3.7.0 -file-read-backwards==2.0.0 -numpy==1.20.2 -inky==1.2.0 -smbus2==0.3.0 -Pillow==5.4.1 -spidev==3.4 -gast==0.2.2 -flask==1.0.2 -flask-cors==3.0.7 -flask-wtf==0.14.3 -dbus-python==1.2.12 -toml==0.10.0 -python-dateutil==2.8.1 -websockets==8.1 - -# flask==1.0.2 requires: -# Jinja2>=2.10 -# Werkzeug>=0.14 -# itsdangerous>=0.24 - -# Jinja2 v2.1.0 requires MarkupSafe>=0.23 for `soft_unicode` -# but the function was removed in MarkupSafe v2.1.0. -# Jinja2 v3.0.0 fixed this issue but flask doesn't support it until v2.0. -MarkupSafe<2.1.0 +# If you get "error: no such option: --prefer-binary" then you need to run: +# pip3 install --upgrade "pip>=20.2" +--prefer-binary +--extra-index-url "https://www.piwheels.org/simple" + +# Used for bluetooth tethering plugin. +dbus-python~=1.2 + +# Used for parsing LastSession logs in manual mode. +file-read-backwards~=2.0 + +# Only using basic Flask and Flask plugin features. +# Should be kept up-to-date as Flask is notorious for breaking +# environments with their extremely loose dependency definitions. +flask-cors~=3.0 +flask-wtf~=1.0 +flask~=1.0 + +# Used for modeling AI parameters. +# NOTE: stable-baselines wants gym[atari,classic_control] but we +# can't satisfy the "atari" extra because it requires ale-py +# which has no source distributions or RasPi wheels. +# Using pip's new backtracking resolver from pip>=20.3 is required +# as it improves handling of extras required by indirect dependencies. +# NOTE: gym v0.22 modified the gym.Env API. +gym~=0.14,<0.22 + +# Used for Inky pHAT and wHAT displays. +inky~=1.2 + +# Used in the AI and UI layers. +# Only using basic numpy features. +numpy~=1.20 + +# Used in the UI layer. +# Only using core PIL features (Image, ImageFont, ImageDraw). +# Very stable library, should be safe to upgrade. +Pillow>=5.4 + +# Used for pwngrid identity verification (PKCS1, RSA, SHA256). +# Very stable library, should be safe to upgrade. +pycryptodome~=3.9 -# Werkzeug v2.2.0 adds a requirement of MarkupSafe>=2.1.1 -# which triggers the `soft_unicode` issue in Jinja2. -# Werkzeug v2.1.0 removes `safe_str_cmp` which breaks flask-wtf. -# flask-wtf fixes this in v0.15. -Werkzeug<2.1.0 +# Used for GPS plugin to parse a GPS datetime string. +python-dateutil~=2.8 -# flask requires itsdangerous>=0.24 for its `json` module -# but the module was removed in v2.1.0. -# flask fixed this issue in v2.X. -itsdangerous<2.1.0 \ No newline at end of file +# Used exclusively to convert legacy YAML configs to TOML. +PyYAML>=5.3 + +# Used for HTTP requests with bettercap, pwngrid, and plugins. +# Only using core library features (GET, POST, Sessions). +# Very stable library, should be safe to upgrade. +requests~=2.21 + +# Used for WiFi pwnage and WiGLE plugin. +scapy~=2.4 + +# I2C/SPI communication with displays, also used by some plugins. +smbus2~=0.4 +spidev~=3.5 + +# Primary AI library. Safe to upgrade as v3 is a different package. +# Upgrading to stable-baselines3 is currently impossible because +# it depends on PyTorch which requires a 64-bit processor. +stable-baselines~=2.7 + +# stable-baselines made a mistake. +# stable-baselines has a tensorflow requirement of ">=1.8.0,<2.0.0", +# but the requirement is the result of a calculation during setup. +# As a result, the requirement is entirely missing from the wheel file. +# Furthermore, "<2.0.0" will fail because tensorflow v1.14 contains +# breaking API changes in preparation for their v2.X release. +tensorflow>=1.8.0,<1.14.0 + +# Used for loading and writing configs. +toml~=0.10 + +# Used for communicating with bettercap. +websockets~=8.1 + +# WARNING: conflict prevention hack! +# flask v1.X requires "Jinja2 >= 2.10, < 3.0" +# Jinja2 v2.X requires "MarkupSafe >= 0.23" for a deprecated +# function that was later removed in MarkupSafe v2.1.0. +# Jinja2 v3.0 no longer uses the deprecated function but +# falls outside the version range requested by flask. +MarkupSafe<2.1.0 From 339a219b976de0389aa5d9699713f60418a8d6df Mon Sep 17 00:00:00 2001 From: llamasoft Date: Tue, 6 Sep 2022 02:07:22 +0000 Subject: [PATCH 12/15] Add pip-compile'd requirements.txt In theory, this file shouldn't need to be regenerated very often. One thing of note is that pip-compile strongly recommends that it be run on the target system in order to ensure it selects the correct versions. This is because pip-compile doesn't (yet) have a way to specify the Python version, platform, and machine type during compilation. The setup.py file was also tweaked to ignore any --options in the requirements.txt file. Signed-off-by: llamasoft --- requirements.txt | 234 ++++++++++++++++++++++++++++++++++++++--------- setup.py | 6 +- 2 files changed, 197 insertions(+), 43 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2730513c8..9dd74a432 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,45 +1,195 @@ -pycryptodome==3.9.4 -requests==2.21.0 -PyYAML==5.3.1 -scapy==2.4.3 -gym==0.14.0 -scipy==1.3.1 -stable-baselines==2.7.0 +# +# This file is autogenerated by pip-compile with python 3.7 +# To update, run: +# +# pip-compile --resolver=backtracking --strip-extras +# +--extra-index-url https://www.piwheels.org/simple + +absl-py==1.2.0 + # via + # tensorboard + # tensorflow +astor==0.8.1 + # via tensorflow +atari-py==0.2.6 + # via gym +certifi==2022.9.24 + # via requests +charset-normalizer==2.1.1 + # via requests +click==7.1.2 + # via flask +cloudpickle==1.6.0 + # via + # gym + # stable-baselines +cycler==0.11.0 + # via matplotlib +dbus-python==1.3.2 + # via -r requirements.in +file-read-backwards==2.0.0 + # via -r requirements.in +flask==1.1.4 + # via + # -r requirements.in + # flask-cors + # flask-wtf +flask-cors==3.0.10 + # via -r requirements.in +flask-wtf==1.0.1 + # via -r requirements.in +fonttools==4.37.4 + # via matplotlib +gast==0.5.3 + # via tensorflow +google-pasta==0.2.0 + # via tensorflow +grpcio==1.49.1 + # via + # tensorboard + # tensorflow +gym==0.19.0 + # via + # -r requirements.in + # stable-baselines +h5py==3.7.0 + # via keras-applications +idna==3.4 + # via requests +importlib-metadata==5.0.0 + # via + # gym + # markdown +inky==1.4.0 + # via -r requirements.in +itsdangerous==1.1.0 + # via + # flask + # flask-wtf +jinja2==2.11.3 + # via flask +joblib==1.2.0 + # via stable-baselines +keras-applications==1.0.8 + # via tensorflow +keras-preprocessing==1.1.2 + # via tensorflow +kiwisolver==1.4.4 + # via matplotlib +markdown==3.4.1 + # via tensorboard +markupsafe==2.0.1 + # via + # -r requirements.in + # jinja2 + # wtforms +matplotlib==3.5.3 + # via stable-baselines +numpy==1.21.4 + # via + # -r requirements.in + # atari-py + # gym + # h5py + # inky + # keras-applications + # keras-preprocessing + # matplotlib + # opencv-python + # pandas + # scipy + # stable-baselines + # tensorboard + # tensorflow +opencv-python==4.6.0.66 + # via + # gym + # stable-baselines +packaging==21.3 + # via matplotlib +pandas==1.3.5 + # via stable-baselines +pillow==9.2.0 + # via + # -r requirements.in + # matplotlib +protobuf==4.21.7 + # via + # tensorboard + # tensorflow +pycryptodome==3.15.0 + # via -r requirements.in +pyglet==1.5.27 + # via gym +pyparsing==3.0.9 + # via + # matplotlib + # packaging +python-dateutil==2.8.2 + # via + # -r requirements.in + # matplotlib + # pandas +pytz==2022.4 + # via pandas +pyyaml==6.0 + # via -r requirements.in +requests==2.28.1 + # via -r requirements.in +scapy==2.4.5 + # via -r requirements.in +scipy==1.7.3 + # via stable-baselines +six==1.16.0 + # via + # atari-py + # flask-cors + # google-pasta + # grpcio + # keras-preprocessing + # python-dateutil + # tensorboard + # tensorflow +smbus2==0.4.2 + # via + # -r requirements.in + # inky +spidev==3.5 + # via + # -r requirements.in + # inky +stable-baselines==2.10.2 + # via -r requirements.in +tensorboard==1.13.1 + # via tensorflow tensorflow==1.13.1 + # via -r requirements.in tensorflow-estimator==1.14.0 -tweepy==3.7.0 -file-read-backwards==2.0.0 -numpy==1.20.2 -inky==1.2.0 -smbus2==0.3.0 -Pillow==5.4.1 -spidev==3.4 -gast==0.2.2 -flask==1.0.2 -flask-cors==3.0.7 -flask-wtf==0.14.3 -dbus-python==1.2.12 -toml==0.10.0 -python-dateutil==2.8.1 + # via tensorflow +termcolor==2.0.1 + # via tensorflow +toml==0.10.2 + # via -r requirements.in +typing-extensions==4.3.0 + # via + # importlib-metadata + # kiwisolver +urllib3==1.26.12 + # via requests websockets==8.1 - -# flask==1.0.2 requires: -# Jinja2>=2.10 -# Werkzeug>=0.14 -# itsdangerous>=0.24 - -# Jinja2 v2.1.0 requires MarkupSafe>=0.23 for `soft_unicode` -# but the function was removed in MarkupSafe v2.1.0. -# Jinja2 v3.0.0 fixed this issue but flask doesn't support it until v2.0. -MarkupSafe<2.1.0 - -# Werkzeug v2.2.0 adds a requirement of MarkupSafe>=2.1.1 -# which triggers the `soft_unicode` issue in Jinja2. -# Werkzeug v2.1.0 removes `safe_str_cmp` which breaks flask-wtf. -# flask-wtf fixes this in v0.15. -Werkzeug<2.1.0 - -# flask requires itsdangerous>=0.24 for its `json` module -# but the module was removed in v2.1.0. -# flask fixed this issue in v2.X. -itsdangerous<2.1.0 \ No newline at end of file + # via -r requirements.in +werkzeug==1.0.1 + # via + # flask + # tensorboard +wheel==0.37.1 + # via + # tensorboard + # tensorflow +wrapt==1.14.1 + # via tensorflow +wtforms==3.0.1 + # via flask-wtf +zipp==3.8.1 + # via importlib-metadata diff --git a/setup.py b/setup.py index 7027f424e..cf057cb03 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,11 @@ def version(version_file): return None with open('requirements.txt') as fp: - required = [line.strip() for line in fp if line.strip() != ""] + required = [ + line.strip() + for line in fp + if line.strip() and not line.startswith("--") + ] VERSION_FILE = 'pwnagotchi/_version.py' pwnagotchi_version = version(VERSION_FILE) From 2d2ccd8560b895a5e3342f9bb78d5dea9911a0bb Mon Sep 17 00:00:00 2001 From: llamasoft Date: Thu, 6 Oct 2022 19:29:41 +0000 Subject: [PATCH 13/15] Add libgtk-3-0 for opencv Despite the fact that a headless version of opencv exists, stable-baselines and gym both request the regular opencv which has a dependency on libgtk3 despite the fact that they don't use any of the GUI functionality. Signed-off-by: llamasoft --- builder/pwnagotchi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/pwnagotchi.yml b/builder/pwnagotchi.yml index 51079803d..572cdb263 100644 --- a/builder/pwnagotchi.yml +++ b/builder/pwnagotchi.yml @@ -74,6 +74,7 @@ - libopenmpi-dev - libatlas-base-dev - libjasper-dev + - libgtk-3-0 - libqtgui4 - libqt4-test - libopenjp2-7 From c63cbe0e8d57cd4ecc546c689ced748713ab5dc3 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Thu, 8 Sep 2022 00:11:55 +0000 Subject: [PATCH 14/15] Install pwnagotchi from local sdist Installing pwnagotchi using the local sdist has a few key benefits: - We're no longer installing all of the data files twice, once in packer and once in the setup.py script. - The image is built using the code in the current local repo, not whatever code is currently pushed to the remote master branch. This makes it much easier to create and test custom images. - Installs pwnagotchi using pip, just like the auto-update plugin does. Signed-off-by: llamasoft --- Makefile | 18 ++++++--- builder/pwnagotchi.json | 82 +++++------------------------------------ builder/pwnagotchi.yml | 55 ++++++++------------------- 3 files changed, 39 insertions(+), 116 deletions(-) diff --git a/Makefile b/Makefile index 4bac72cc1..78d722c3e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PACKER_VERSION := 1.8.3 PWN_HOSTNAME := pwnagotchi -PWN_VERSION := master +PWN_VERSION := $(shell cut -d"'" -f2 < pwnagotchi/_version.py) PWN_RELEASE := pwnagotchi-raspios-lite-$(PWN_VERSION) MACHINE_TYPE := $(shell uname -m) @@ -43,11 +43,15 @@ $(PACKER): rm $(PACKER).zip chmod +x $@ +SDIST := dist/pwnagotchi-$(PWN_VERSION).tar.gz +$(SDIST): setup.py pwnagotchi + python3 setup.py sdist + # Building the image requires packer, but don't rebuild the image just because packer updated. $(PWN_RELEASE).img: | $(PACKER) # If the packer or ansible files are updated, rebuild the image. -$(PWN_RELEASE).img: builder/pwnagotchi.json builder/pwnagotchi.yml $(shell find builder/data -type f) +$(PWN_RELEASE).img: $(SDIST) builder/pwnagotchi.json builder/pwnagotchi.yml $(shell find builder/data -type f) sudo $(PACKER) plugins install github.com/solo-io/arm-image cd builder && sudo $(UNSHARE) $(PACKER) build -var "pwn_hostname=$(PWN_HOSTNAME)" -var "pwn_version=$(PWN_VERSION)" pwnagotchi.json sudo chown -R $$USER:$$USER builder/output-pwnagotchi @@ -65,6 +69,10 @@ $(PWN_RELEASE).zip: $(PWN_RELEASE).img $(PWN_RELEASE).sha256 image: $(PWN_RELEASE).zip clean: - rm -f $(PACKER) - rm -f $(PWN_RELEASE).* - sudo rm -rf builder/output-pwnagotchi builder/packer_cache + - python3 setup.py clean --all + - rm -rf dist pwnagotchi.egg-info + - rm -f $(PACKER) + - rm -f $(PWN_RELEASE).* + - sudo rm -rf builder/output-pwnagotchi builder/packer_cache + + diff --git a/builder/pwnagotchi.json b/builder/pwnagotchi.json index 87c0a8e3b..63988bf95 100644 --- a/builder/pwnagotchi.json +++ b/builder/pwnagotchi.json @@ -13,86 +13,24 @@ { "type": "shell", "inline": [ - "sed -i 's/^\\([^#]\\)/#\\1/g' /etc/ld.so.preload", + "mv /etc/ld.so.preload /etc/ld.so.preload.DISABLED", + "uname -a", "dpkg-architecture", - "apt-get -y --allow-releaseinfo-change update", - "apt-get install -y ansible" + "mkdir -p /usr/local/src/pwnagotchi" ] }, { "type": "file", - "source": "data/usr/bin/pwnlib", - "destination": "/usr/bin/pwnlib" - }, - { - "type": "file", - "source": "data/usr/bin/bettercap-launcher", - "destination": "/usr/bin/bettercap-launcher" - }, - { - "type": "file", - "source": "data/usr/bin/pwnagotchi-launcher", - "destination": "/usr/bin/pwnagotchi-launcher" - }, - { - "type": "file", - "source": "data/usr/bin/monstop", - "destination": "/usr/bin/monstop" - }, - { - "type": "file", - "source": "data/usr/bin/monstart", - "destination": "/usr/bin/monstart" - }, - { - "type": "file", - "source": "data/usr/bin/hdmion", - "destination": "/usr/bin/hdmion" - }, - { - "type": "file", - "source": "data/usr/bin/hdmioff", - "destination": "/usr/bin/hdmioff" - }, - { - "type": "file", - "source": "data/etc/network/interfaces.d/lo-cfg", - "destination": "/etc/network/interfaces.d/lo-cfg" - }, - { - "type": "file", - "source": "data/etc/network/interfaces.d/wlan0-cfg", - "destination": "/etc/network/interfaces.d/wlan0-cfg" - }, - { - "type": "file", - "source": "data/etc/network/interfaces.d/usb0-cfg", - "destination": "/etc/network/interfaces.d/usb0-cfg" - }, - { - "type": "file", - "source": "data/etc/network/interfaces.d/eth0-cfg", - "destination": "/etc/network/interfaces.d/eth0-cfg" - }, - { - "type": "file", - "source": "data/etc/systemd/system/pwngrid-peer.service", - "destination": "/etc/systemd/system/pwngrid-peer.service" - }, - { - "type": "file", - "source": "data/etc/systemd/system/pwnagotchi.service", - "destination": "/etc/systemd/system/pwnagotchi.service" - }, - { - "type": "file", - "source": "data/etc/systemd/system/bettercap.service", - "destination": "/etc/systemd/system/bettercap.service" + "sources": [ + "../dist/pwnagotchi-{{user `pwn_version`}}.tar.gz" + ], + "destination": "/usr/local/src/pwnagotchi/" }, { "type": "shell", "inline": [ - "chmod +x /usr/bin/*" + "apt-get -y --allow-releaseinfo-change update", + "apt-get install -y --no-install-recommends ansible" ] }, { @@ -104,7 +42,7 @@ { "type": "shell", "inline": [ - "sed -i 's/^#\\(.+\\)/\\1/g' /etc/ld.so.preload" + "mv /etc/ld.so.preload.DISABLED /etc/ld.so.preload" ] } ] diff --git a/builder/pwnagotchi.yml b/builder/pwnagotchi.yml index 572cdb263..8595b7663 100644 --- a/builder/pwnagotchi.yml +++ b/builder/pwnagotchi.yml @@ -216,25 +216,22 @@ regexp: "#EPD_SIZE=2.0" line: "EPD_SIZE=2.0" - - name: collect python pip package list - command: "pip3 list" - register: pip_output - - - name: set python pip package facts - set_fact: - pip_packages: > - {{ pip_packages | default({}) | combine( { item.split()[0]: item.split()[1] } ) }} - with_items: "{{ pip_output.stdout_lines }}" - - - name: acquire python3 pip target - command: "python3 -c 'import sys;print(sys.path.pop())'" - register: pip_target - - - name: clone pwnagotchi repository - git: - repo: https://github.com/evilsocket/pwnagotchi.git - dest: /usr/local/src/pwnagotchi - register: pwnagotchigit + # pip v20.3 uses a newer dependency resolver that better handles our unique situation. + # Specifically, it handles mismatches between direct requirements without extras and + # indirect requirements that do want extras (e.g. gym vs stable-baselines->gym[atari]). + - name: Upgrade pip + pip: + name: "pip" + version: ">=20.3" + + # We need the --ignore-installed option so that pip simply overwrites/upgrades existing + # packages instead of trying to uninstall them first. While this sounds dangerous, + # this matches the legacy behavior of pip. This is required to prevent pip from trying + # (and failing) to uninstall python packages that were originally installed via apt. + - name: Install pwnagotchi from source archive + pip: + name: /usr/local/src/pwnagotchi/pwnagotchi-{{ pwnagotchi.version }}.tar.gz + extra_args: --verbose --prefer-binary --ignore-installed - name: create /usr/local/share/pwnagotchi/ folder file: @@ -246,26 +243,6 @@ repo: https://github.com/evilsocket/pwnagotchi-plugins-contrib.git dest: /usr/local/share/pwnagotchi/availaible-plugins - - name: fetch pwnagotchi version - set_fact: - pwnagotchi_version: "{{ lookup('file', '/usr/local/src/pwnagotchi/pwnagotchi/_version.py') | regex_replace('.*__version__.*=.*''([0-9]+\\.[0-9]+\\.[0-9]+[A-Za-z0-9]*)''.*', '\\1') }}" - - - name: pwnagotchi version found - debug: - msg: "{{ pwnagotchi_version }}" - - - name: build pwnagotchi wheel - command: "python3 setup.py sdist bdist_wheel" - args: - chdir: /usr/local/src/pwnagotchi - when: (pwnagotchigit.changed) or (pip_packages['pwnagotchi'] is undefined) or (pip_packages['pwnagotchi'] != pwnagotchi_version) - - - name: install pwnagotchi wheel and dependencies - pip: - name: "{{ lookup('fileglob', '/usr/local/src/pwnagotchi/dist/pwnagotchi*.whl') }}" - extra_args: "--no-cache-dir" - when: (pwnagotchigit.changed) or (pip_packages['pwnagotchi'] is undefined) or (pip_packages['pwnagotchi'] != pwnagotchi_version) - - name: download and install pwngrid unarchive: src: "{{ packages.pwngrid.url }}" From 58095c8aee632f1f5df12a17db196109364be900 Mon Sep 17 00:00:00 2001 From: llamasoft Date: Thu, 8 Sep 2022 04:25:41 +0000 Subject: [PATCH 15/15] Actually clean the apt cache As the added comments describe, the options available to us from Ansible's apt module don't actually clean the apt cache. Manually running apt-get clean recovers around 400MB of space. Signed-off-by: llamasoft --- builder/pwnagotchi.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/builder/pwnagotchi.yml b/builder/pwnagotchi.yml index 8595b7663..21a2d4255 100644 --- a/builder/pwnagotchi.yml +++ b/builder/pwnagotchi.yml @@ -375,9 +375,14 @@ You learn more about me at https://pwnagotchi.ai/ when: hostname.changed + # Ansible's apt module has an "autoclean" option but it only removes packages + # that can no longer be downloaded. Ansible v2.13 added the "clean" option + # which actually purges the apt cache, but that's newer than what we can + # install from the RasPiOS repos. Instead, we'll manually clean the cache. - name: clean apt cache - apt: - autoclean: yes + command: "apt-get clean" + args: + warn: false - name: remove dependencies that are no longer required apt: