scraping-worker (latest)
Published 2025-10-16 12:12:19 +00:00 by hari
Installation
docker pull git.camelot.ad/thesis/scraping-worker:latestsha256:e71bcd7ceb1606a3474f47c41a6c1b984337fd0337e21f25a3f681631dd83993Image Layers
| # debian.sh --arch 'amd64' out/ 'trixie' '@1759104000' |
| ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin |
| ENV LANG=C.UTF-8 |
| RUN /bin/sh -c set -eux; apt-get update; apt-get install -y --no-install-recommends ca-certificates netbase tzdata ; apt-get dist-clean # buildkit |
| ENV GPG_KEY=A035C8C19219BA821ECEA86B64E628F8D684696D |
| ENV PYTHON_VERSION=3.10.19 |
| ENV PYTHON_SHA256=c8f4a596572201d81dd7df91f70e177e19a70f1d489968b54b5fbbf29a97c076 |
| RUN /bin/sh -c set -eux; savedAptMark="$(apt-mark showmanual)"; apt-get update; apt-get install -y --no-install-recommends dpkg-dev gcc gnupg libbluetooth-dev libbz2-dev libc6-dev libdb-dev libffi-dev libgdbm-dev liblzma-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev make tk-dev uuid-dev wget xz-utils zlib1g-dev ; wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; echo "$PYTHON_SHA256 *python.tar.xz" | sha256sum -c -; wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; gpg --batch --verify python.tar.xz.asc python.tar.xz; gpgconf --kill all; rm -rf "$GNUPGHOME" python.tar.xz.asc; mkdir -p /usr/src/python; tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; rm python.tar.xz; cd /usr/src/python; gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; ./configure --build="$gnuArch" --enable-loadable-sqlite-extensions --enable-optimizations --enable-option-checking=fatal --enable-shared $(test "${gnuArch%%-*}" != 'riscv64' && echo '--with-lto') --with-ensurepip ; nproc="$(nproc)"; EXTRA_CFLAGS="$(dpkg-buildflags --get CFLAGS)"; LDFLAGS="$(dpkg-buildflags --get LDFLAGS)"; LDFLAGS="${LDFLAGS:--Wl},--strip-all"; make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:-}" ; rm python; make -j "$nproc" "EXTRA_CFLAGS=${EXTRA_CFLAGS:-}" "LDFLAGS=${LDFLAGS:--Wl},-rpath='\$\$ORIGIN/../lib'" python ; make install; cd /; rm -rf /usr/src/python; find /usr/local -depth \( \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \) -exec rm -rf '{}' + ; ldconfig; apt-mark auto '.*' > /dev/null; apt-mark manual $savedAptMark; find /usr/local -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' | awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' | sort -u | xargs -rt dpkg-query --search | awk 'sub(":$", "", $1) { print $1 }' | sort -u | xargs -r apt-mark manual ; apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; apt-get dist-clean; export PYTHONDONTWRITEBYTECODE=1; python3 --version; pip3 install --disable-pip-version-check --no-cache-dir --no-compile 'setuptools==79.0.1' 'wheel<0.46' ; pip3 --version # buildkit |
| RUN /bin/sh -c set -eux; for src in idle3 pip3 pydoc3 python3 python3-config; do dst="$(echo "$src" | tr -d 3)"; [ -s "/usr/local/bin/$src" ]; [ ! -e "/usr/local/bin/$dst" ]; ln -svT "$src" "/usr/local/bin/$dst"; done # buildkit |
| CMD ["python3"] |
| RUN /bin/sh -c apt-get update && apt-get install -y curl xvfb && rm -rf /var/lib/apt/lists/* # buildkit |
| RUN /bin/sh -c pip install --no-cache-dir uv # buildkit |
| WORKDIR /app |
| COPY pyproject.toml ./ # buildkit |
| RUN /bin/sh -c uv pip install --system --no-cache-dir . # buildkit |
| COPY . . # buildkit |
| RUN /bin/sh -c apt-get update && apt-get install -y libglib2.0-0 libnspr4 libnss3 libdbus-1-3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libxkbcommon0 libatspi2.0-0 libxcomposite1 libxdamage1 libxfixes3 libcairo2 libpango-1.0-0 libasound2 fonts-unifont fonts-liberation && rm -rf /var/lib/apt/lists/* && playwright install chromium # buildkit |
| ENV DISPLAY=:99 |
| ENV PYTHONPATH=/app |
| ENV PYTHONUNBUFFERED=1 |
| ENV MAX_TASK_RETRIES=3 |
| ENV RETRY_DELAY_MINUTES=5 |
| ENV MAX_DEAD_LETTER_AGE_HOURS=24 |
| RUN /bin/sh -c mkdir -p /app/scraped_html # buildkit |
| CMD ["/bin/bash" "-c" "rm -f /tmp/.X99-lock; Xvfb :99 -screen 0 1920x1080x24 -ac +extension GLX +render -noreset & export DISPLAY=:99 && sleep 3 && python worker_enhanced_safe.py"] |