mirror/naiveproxy
1
0
Fork 0
mirror of https://github.com/klzgrad/naiveproxy.git synced 2024-11-24 14:26:09 +03:00
Code Issues Projects Releases Wiki Activity

build: Minimize sysroot creator script

Browse Source
This commit is contained in:
klzgrad 2019-10-05 15:52:34 +08:00
parent 0ca466521a
commit 3d50fcde3b
5 changed files with 186 additions and 493 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/build/linux/sysroot_scripts/generate_keyring.sh
View File

@ -9,8 +9,14 @@ set -o errexit
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
KEYS=(
# Debian Ports Archive Automatic Signing Key (2023)
"B523E5F3FC4E5F2C"
# Debian Ports Archive Automatic Signing Key (2024)
"8D69674688B6CB36"
# Debian Archive Automatic Signing Key (12/bookworm)
"6ED0E7B82643E131"
"B7C5D7D6350947F8"
# Debian Security Archive Automatic Signing Key (12/bookworm)
"254CF3B5AEC0A8F0"
# Debian Stable Release Key (12/bookworm)
"F8D2585B8783D481"
# Debian Archive Automatic Signing Key (11/bullseye)

2
src/build/linux/sysroot_scripts/install-sysroot.py
View File

@ -36,7 +36,7 @@ from urllib.request import urlopen
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
VALID_ARCHS = ("amd64", "i386", "armhf", "arm64", "mipsel", "mips64el")
VALID_ARCHS = ("amd64", "i386", "armhf", "arm64", "mipsel", "mips64el", "riscv64")
ARCH_TRANSLATIONS = {
"x64": "amd64",

125
src/build/linux/sysroot_scripts/reversion_glibc.py Normal file
View File

@ -0,0 +1,125 @@
# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Rewrite incompatible default symbols in glibc.
"""
import re
import subprocess
# This constant comes from the oldest glibc version in
# //chrome/installer/linux/debian/dist_package_versions.json and
# //chrome/installer/linux/rpm/dist_package_provides.json
MAX_ALLOWED_GLIBC_VERSION = [2, 26]
VERSION_PATTERN = re.compile(r"GLIBC_([0-9\.]+)")
SECTION_PATTERN = re.compile(r"^ *\[ *[0-9]+\] +(\S+) +\S+ + ([0-9a-f]+) .*$")
# Some otherwise disallowed symbols are referenced in the linux-chromeos build.
# To continue supporting it, allow these symbols to remain enabled.
SYMBOL_ALLOWLIST = {
"fts64_close",
"fts64_open",
"fts64_read",
"memfd_create",
}
def reversion_glibc(bin_file: str) -> None:
# The two dictionaries below map from symbol name to
# (symbol version, symbol index).
#
# The default version for a given symbol (which may be unsupported).
default_version = {}
# The max supported symbol version for a given symbol.
supported_version = {}
# Populate |default_version| and |supported_version| with data from readelf.
stdout = subprocess.check_output(
["readelf", "--dyn-syms", "--wide", bin_file])
for line in stdout.decode("utf-8").split("\n"):
cols = re.split(r"\s+", line)
# Skip the preamble.
if len(cols) < 9:
continue
index = cols[1].rstrip(":")
# Skip the header.
if not index.isdigit():
continue
index = int(index)
name = cols[8].split("@")
# Ignore unversioned symbols.
if len(name) < 2:
continue
base_name = name[0]
version = name[-1]
# The default version will have '@@' in the name.
is_default = len(name) > 2
if version.startswith("XCRYPT_"):
# Prefer GLIBC_* versioned symbols over XCRYPT_* ones.
# Set the version to something > MAX_ALLOWED_GLIBC_VERSION
# so this symbol will not be picked.
version = [10**10]
else:
match = re.match(VERSION_PATTERN, version)
# Ignore symbols versioned with GLIBC_PRIVATE.
if not match:
continue
version = [int(part) for part in match.group(1).split(".")]
if version < MAX_ALLOWED_GLIBC_VERSION:
old_supported_version = supported_version.get(
base_name, ([-1], -1))
supported_version[base_name] = max((version, index),
old_supported_version)
if is_default:
default_version[base_name] = (version, index)
# Get the offset into the binary of the .gnu.version section from readelf.
stdout = subprocess.check_output(
["readelf", "--sections", "--wide", bin_file])
for line in stdout.decode("utf-8").split("\n"):
if match := SECTION_PATTERN.match(line):
section_name, address = match.groups()
if section_name == ".gnu.version":
gnu_version_addr = int(address, base=16)
break
else:
raise Exception("No .gnu.version section found")
# Rewrite the binary.
bin_data = bytearray(open(bin_file, "rb").read())
for name, (version, index) in default_version.items():
# No need to rewrite the default if it's already an allowed version.
if version <= MAX_ALLOWED_GLIBC_VERSION:
continue
if name in SYMBOL_ALLOWLIST:
continue
elif name in supported_version:
_, supported_index = supported_version[name]
else:
supported_index = -1
# The .gnu.version section is divided into 16-bit chunks that give the
# symbol versions. The 16th bit is a flag that's false for the default
# version. The data is stored in little-endian so we need to add 1 to
# get the address of the byte we want to flip.
#
# Disable the unsupported symbol.
old_default = gnu_version_addr + 2 * index + 1
assert (bin_data[old_default] & 0x80) == 0
bin_data[old_default] ^= 0x80
# If we found a supported version, enable that as default.
if supported_index != -1:
new_default = gnu_version_addr + 2 * supported_index + 1
assert (bin_data[new_default] & 0x80) == 0x80
bin_data[new_default] ^= 0x80
open(bin_file, "wb").write(bin_data)

538
src/build/linux/sysroot_scripts/sysroot_creator.py
View File

@ -17,6 +17,7 @@ import tempfile
import time
import requests
import reversion_glibc
DISTRO = "debian"
RELEASE = "bullseye"
@ -38,10 +39,6 @@ ARCHIVE_TIMESTAMP = "20230611T210420Z"
ARCHIVE_URL = f"https://snapshot.debian.org/archive/debian/{ARCHIVE_TIMESTAMP}/"
APT_SOURCES_LIST = [
# Debian 12 (Bookworm) is needed for GTK4. It should be kept before
# bullseye so that bullseye takes precedence.
("bookworm", ["main"]),
("bookworm-updates", ["main"]),
# This mimics a sources.list from bullseye.
("bullseye", ["main", "contrib", "non-free"]),
("bullseye-updates", ["main", "contrib", "non-free"]),
@ -55,6 +52,7 @@ TRIPLES = {
"arm64": "aarch64-linux-gnu",
"mipsel": "mipsel-linux-gnu",
"mips64el": "mips64el-linux-gnuabi64",
"riscv64": "riscv64-linux-gnu",
}
REQUIRED_TOOLS = [
@ -73,520 +71,48 @@ RELEASE_FILE_GPG = "Release.gpg"
# Packages common to all architectures.
DEBIAN_PACKAGES = [
"comerr-dev",
"krb5-multidev",
"libaom0",
"libaom3",
"libasound2",
"libasound2-dev",
"libasyncns0",
"libatk-bridge2.0-0",
"libatk-bridge2.0-dev",
"libatk1.0-0",
"libatk1.0-dev",
"libatomic1",
"libatspi2.0-0",
"libatspi2.0-dev",
"libattr1",
"libaudit1",
"libavahi-client3",
"libavahi-common3",
"libavcodec-dev",
"libavcodec58",
"libavcodec59",
"libavformat-dev",
"libavformat59",
"libavutil-dev",
"libavutil56",
"libavutil57",
"libb2-1",
"libblkid-dev",
"libblkid1",
"libbluetooth-dev",
"libbluetooth3",
"libbluray2",
"libbrotli-dev",
"libbrotli1",
"libbsd0",
"libbz2-1.0",
"libc6",
"libc6-dev",
"libcairo-gobject2",
"libcairo-script-interpreter2",
"libcairo2",
"libcairo2-dev",
"libcap-dev",
"libcap-ng0",
"libcap2",
"libchromaprint1",
"libcjson1",
"libcloudproviders0",
"libcodec2-0.9",
"libcodec2-1.0",
"libcolord2",
"libcom-err2",
"libcrypt-dev",
"libcrypt1",
"libcups2",
"libcups2-dev",
"libcupsimage2",
"libcupsimage2-dev",
"libcurl3-gnutls",
"libcurl4-gnutls-dev",
"libdatrie-dev",
"libdatrie1",
"libdav1d4",
"libdav1d6",
"libdb5.3",
"libdbus-1-3",
"libdbus-1-dev",
"libdbus-glib-1-2",
"libdbusmenu-glib-dev",
"libdbusmenu-glib4",
"libdbusmenu-gtk3-4",
"libdbusmenu-gtk4",
"libdeflate-dev",
"libdeflate0",
"libdouble-conversion3",
"libdrm-amdgpu1",
"libdrm-dev",
"libdrm-nouveau2",
"libdrm-radeon1",
"libdrm2",
"libegl-dev",
"libegl1",
"libegl1-mesa",
"libegl1-mesa-dev",
"libelf-dev",
"libelf1",
"libepoxy-dev",
"libepoxy0",
"libevdev-dev",
"libevdev2",
"libevent-2.1-7",
"libexpat1",
"libexpat1-dev",
"libffi-dev",
"libffi7",
"libflac-dev",
"libflac8",
"libfontconfig-dev",
"libfontconfig1",
"libfreetype-dev",
"libfreetype6",
"libfribidi-dev",
"libfribidi0",
"libgbm-dev",
"libgbm1",
"libgcc-10-dev",
"libgcc-s1",
"libgcrypt20",
"libgcrypt20-dev",
"libgdk-pixbuf-2.0-0",
"libgdk-pixbuf-2.0-dev",
"libgl-dev",
"libgl1",
"libgl1-mesa-dev",
"libgl1-mesa-glx",
"libglapi-mesa",
"libgles-dev",
"libgles1",
"libgles2",
"libglib2.0-0",
"libglib2.0-dev",
"libglvnd-dev",
"libglvnd0",
"libglx-dev",
"libglx0",
"libgme0",
"libgmp10",
"libgnutls-dane0",
"libgnutls-openssl27",
"libgnutls28-dev",
"libgnutls30",
"libgnutlsxx28",
"libgomp1",
"libgpg-error-dev",
"libgpg-error0",
"libgraphene-1.0-0",
"libgraphene-1.0-dev",
"libgraphite2-3",
"libgraphite2-dev",
"libgsm1",
"libgssapi-krb5-2",
"libgssrpc4",
"libgtk-3-0",
"libgtk-3-dev",
"libgtk-4-1",
"libgtk-4-dev",
"libgtk2.0-0",
"libgudev-1.0-0",
"libharfbuzz-dev",
"libharfbuzz-gobject0",
"libharfbuzz-icu0",
"libharfbuzz0b",
"libhogweed6",
"libhwy1",
"libice6",
"libicu-le-hb0",
"libicu67",
"libidl-2-0",
"libidn11",
"libidn2-0",
"libinput-dev",
"libinput10",
"libjbig-dev",
"libjbig0",
"libjpeg62-turbo",
"libjpeg62-turbo-dev",
"libjson-glib-1.0-0",
"libjsoncpp-dev",
"libjsoncpp24",
"libjxl0.7",
"libjxl0.7",
"libk5crypto3",
"libkadm5clnt-mit12",
"libkadm5srv-mit12",
"libkdb5-10",
"libkeyutils1",
"libkrb5-3",
"libkrb5-dev",
"libkrb5support0",
"liblcms2-2",
"libldap-2.4-2",
"liblerc4",
"libltdl7",
"liblz4-1",
"liblzma5",
"liblzo2-2",
"libmbedcrypto7",
"libmd0",
"libmd4c0",
"libminizip-dev",
"libminizip1",
"libmount-dev",
"libmount1",
"libmp3lame0",
"libmpg123-0",
"libmtdev1",
"libncurses-dev",
"libncurses6",
"libncursesw6",
"libnettle8",
"libnghttp2-14",
"libnorm1",
"libnsl2",
"libnspr4",
"libnspr4-dev",
"libnss-db",
"libnss3",
"libnss3-dev",
"libnuma1",
"libogg-dev",
"libogg0",
"libopengl0",
"libopenjp2-7",
"libopenmpt0",
"libopus-dev",
"libopus-dev",
"libopus0",
"libp11-kit0",
"libpam0g",
"libpam0g-dev",
"libpango-1.0-0",
"libpango1.0-dev",
"libpangocairo-1.0-0",
"libpangoft2-1.0-0",
"libpangox-1.0-0",
"libpangoxft-1.0-0",
"libpci-dev",
"libpci3",
"libpciaccess0",
"libpcre16-3",
"libpcre2-16-0",
"libpcre2-32-0",
"libpcre2-8-0",
"libpcre2-dev",
"libpcre2-posix2",
"libpcre3",
"libpcre3-dev",
"libpcre32-3",
"libpcrecpp0v5",
"libpgm-5.3-0",
"libpipewire-0.3-0",
"libpipewire-0.3-dev",
"libpixman-1-0",
"libpixman-1-dev",
"libpng-dev",
"libpng16-16",
"libproxy1v5",
"libpsl5",
"libpthread-stubs0-dev",
"libpulse-dev",
"libpulse-mainloop-glib0",
"libpulse0",
"libqt5concurrent5",
"libqt5core5a",
"libqt5dbus5",
"libqt5gui5",
"libqt5network5",
"libqt5printsupport5",
"libqt5sql5",
"libqt5test5",
"libqt5widgets5",
"libqt5xml5",
"libqt6concurrent6",
"libqt6core6",
"libqt6dbus6",
"libqt6gui6",
"libqt6network6",
"libqt6opengl6",
"libqt6openglwidgets6",
"libqt6printsupport6",
"libqt6sql6",
"libqt6test6",
"libqt6widgets6",
"libqt6xml6",
"librabbitmq4",
"librav1e0",
"libre2-9",
"libre2-dev",
"librest-0.7-0",
"librist4",
"librsvg2-2",
"librtmp1",
"libsasl2-2",
"libselinux1",
"libselinux1-dev",
"libsepol1",
"libsepol1-dev",
"libshine3",
"libsm6",
"libsnappy-dev",
"libsnappy1v5",
"libsndfile1",
"libsodium23",
"libsoup-gnome2.4-1",
"libsoup2.4-1",
"libsoxr0",
"libspa-0.2-dev",
"libspeechd-dev",
"libspeechd2",
"libspeex1",
"libsqlite3-0",
"libsrt1.5-gnutls",
"libssh-gcrypt-4",
"libssh2-1",
"libssl-dev",
"libssl1.1",
"libstdc++-10-dev",
"libstdc++6",
"libsvtav1enc1",
"libswresample-dev",
"libswresample3",
"libswresample4",
"libsystemd-dev",
"libsystemd0",
"libtasn1-6",
"libthai-dev",
"libthai0",
"libtheora0",
"libtheora0",
"libtiff-dev",
"libtiff5",
"libtiff6",
"libtiffxx5",
"libtinfo6",
"libtirpc3",
"libts0",
"libtwolame0",
"libudev-dev",
"libudev1",
"libudfread0",
"libunbound8",
"libunistring2",
"libutempter-dev",
"libutempter0",
"libuuid1",
"libva-dev",
"libva-drm2",
"libva-glx2",
"libva-wayland2",
"libva-x11-2",
"libva2",
"libvdpau1",
"libvorbis0a",
"libvorbisenc2",
"libvorbisfile3",
"libvpx-dev",
"libvpx6",
"libvpx7",
"libvulkan-dev",
"libvulkan1",
"libwacom2",
"libwavpack1",
"libwayland-bin",
"libwayland-client0",
"libwayland-cursor0",
"libwayland-dev",
"libwayland-egl-backend-dev",
"libwayland-egl1",
"libwayland-egl1-mesa",
"libwayland-server0",
"libwebp-dev",
"libwebp6",
"libwebp7",
"libwebpdemux2",
"libwebpmux3",
"libwrap0",
"libx11-6",
"libx11-dev",
"libx11-xcb-dev",
"libx11-xcb1",
"libx264-160",
"libx264-164",
"libx265-192",
"libx265-199",
"libxau-dev",
"libxau6",
"libxcb-dri2-0",
"libxcb-dri2-0-dev",
"libxcb-dri3-0",
"libxcb-dri3-dev",
"libxcb-glx0",
"libxcb-glx0-dev",
"libxcb-icccm4",
"libxcb-image0",
"libxcb-image0-dev",
"libxcb-keysyms1",
"libxcb-present-dev",
"libxcb-present0",
"libxcb-randr0",
"libxcb-randr0-dev",
"libxcb-render-util0",
"libxcb-render-util0-dev",
"libxcb-render0",
"libxcb-render0-dev",
"libxcb-shape0",
"libxcb-shape0-dev",
"libxcb-shm0",
"libxcb-shm0-dev",
"libxcb-sync-dev",
"libxcb-sync1",
"libxcb-util-dev",
"libxcb-util1",
"libxcb-xfixes0",
"libxcb-xfixes0-dev",
"libxcb-xinerama0",
"libxcb-xinput0",
"libxcb-xkb1",
"libxcb1",
"libxcb1-dev",
"libxcomposite-dev",
"libxcomposite1",
"libxcursor-dev",
"libxcursor1",
"libxdamage-dev",
"libxdamage1",
"libxdmcp-dev",
"libxdmcp6",
"libxext-dev",
"libxext6",
"libxfixes-dev",
"libxfixes3",
"libxft-dev",
"libxft2",
"libxi-dev",
"libxi6",
"libxinerama-dev",
"libxinerama1",
"libxkbcommon-dev",
"libxkbcommon-x11-0",
"libxkbcommon0",
"libxml2",
"libxml2-dev",
"libxrandr-dev",
"libxrandr2",
"libxrender-dev",
"libxrender1",
"libxshmfence-dev",
"libxshmfence1",
"libxslt1-dev",
"libxslt1.1",
"libxss-dev",
"libxss1",
"libxt-dev",
"libxt6",
"libxtst-dev",
"libxtst6",
"libxvidcore4",
"libxxf86vm-dev",
"libxxf86vm1",
"libzmq5",
"libzstd1",
"libzvbi0",
"linux-libc-dev",
"mesa-common-dev",
"ocl-icd-libopencl1",
"qt6-base-dev",
"qt6-base-dev-tools",
"qtbase5-dev",
"qtbase5-dev-tools",
"shared-mime-info",
"uuid-dev",
"wayland-protocols",
"x11proto-dev",
"zlib1g",
"zlib1g-dev",
]
DEBIAN_PACKAGES_ARCH = {
"amd64": [
"libasan6",
"libdrm-intel1",
"libitm1",
"liblsan0",
"libmfx1",
"libquadmath0",
"libtsan0",
"libubsan1",
"valgrind",
],
"i386": [
"libasan6",
"libdrm-intel1",
"libitm1",
"libquadmath0",
"libubsan1",
"valgrind",
],
"armhf": [
"libasan6",
"libdrm-etnaviv1",
"libdrm-exynos1",
"libdrm-freedreno1",
"libdrm-omap1",
"libdrm-tegra0",
"libubsan1",
"valgrind",
],
"arm64": [
"libasan6",
"libdrm-etnaviv1",
"libdrm-freedreno1",
"libdrm-tegra0",
"libgmp10",
"libitm1",
"liblsan0",
"libthai0",
"libtsan0",
"libubsan1",
"valgrind",
],
"mipsel": [],
"mips64el": [
"valgrind",
],
"mips64el": [],
"riscv64": [],
}
@ -728,11 +254,13 @@ def generate_package_list_dist_repo(arch: str, dist: str,
download_or_copy_non_unique_filename(package_list_arch, package_list)
verify_package_listing(package_file_arch, package_list, dist)
# `not line.endswith(":")` is added here to handle the case of
# "X-Cargo-Built-Using:\n rust-adler (= 1.0.2-2), ..."
with lzma.open(package_list, "rt") as src:
return [
dict(
line.split(": ", 1) for line in package_meta.splitlines()
if not line.startswith(" "))
if not line.startswith(" ") and not line.endswith(":"))
for package_meta in src.read().split("\n\n") if package_meta
]
@ -774,13 +302,34 @@ def hacks_and_patches(install_root: str, script_dir: str, arch: str) -> None:
if os.path.exists(qtchooser_conf):
os.remove(qtchooser_conf)
# libxcomposite1 is missing a symbols file.
atomic_copyfile(
os.path.join(script_dir, "libxcomposite1-symbols"),
os.path.join(install_root, "debian", "libxcomposite1", "DEBIAN",
"symbols"),
# __GLIBC_MINOR__ is used as a feature test macro. Replace it with the
# earliest supported version of glibc (2.26).
features_h = os.path.join(install_root, "usr", "include", "features.h")
replace_in_file(features_h, r"(#define\s+__GLIBC_MINOR__)", r"\1 26 //")
# fcntl64() was introduced in glibc 2.28. Make sure to use fcntl() instead.
fcntl_h = os.path.join(install_root, "usr", "include", "fcntl.h")
replace_in_file(
fcntl_h,
r"#ifndef __USE_FILE_OFFSET64(\nextern int fcntl)",
r"#if 1\1",
)
# Do not use pthread_cond_clockwait as it was introduced in glibc 2.30.
cppconfig_h = os.path.join(
install_root,
"usr",
"include",
TRIPLES[arch],
"c++",
"10",
"bits",
"c++config.h",
)
replace_in_file(cppconfig_h,
r"(#define\s+_GLIBCXX_USE_PTHREAD_COND_CLOCKWAIT)",
r"// \1")
# Include limits.h in stdlib.h to fix an ODR issue.
stdlib_h = os.path.join(install_root, "usr", "include", "stdlib.h")
replace_in_file(stdlib_h, r"(#include <stddef.h>)",
@ -796,12 +345,14 @@ def hacks_and_patches(install_root: str, script_dir: str, arch: str) -> None:
shutil.move(os.path.join(triple_pkgconfig_dir, file),
pkgconfig_dir)
# GTK4 is provided by bookworm (12), but pango is provided by bullseye
# (11). Fix the GTK4 pkgconfig file to relax the pango version
# requirement.
gtk4_pc = os.path.join(pkgconfig_dir, "gtk4.pc")
replace_in_file(gtk4_pc, r"pango [>=0-9. ]*", "pango")
replace_in_file(gtk4_pc, r"pangocairo [>=0-9. ]*", "pangocairo")
# Avoid requiring unsupported glibc versions.
for lib in ["libc.so.6", "libm.so.6", "libcrypt.so.1"]:
# RISCV64 is new and has no backward compatibility.
# Reversioning would remove necessary symbols and cause linking failures.
if arch == "riscv64":
continue
lib_path = os.path.join(install_root, "lib", TRIPLES[arch], lib)
reversion_glibc.reversion_glibc(lib_path)
def replace_in_file(file_path: str, search_pattern: str,
@ -952,7 +503,6 @@ def build_sysroot(arch: str) -> None:
hacks_and_patches(install_root, SCRIPT_DIR, arch)
cleanup_jail_symlinks(install_root)
verify_library_deps(install_root)
create_tarball(install_root, arch)
def upload_sysroot(arch: str) -> str:
@ -1019,6 +569,12 @@ def main():
sanity_check()
# RISCV64 only has support in debian-ports, no support in bookworm.
if args.architecture == "riscv64":
global ARCHIVE_URL, APT_SOURCES_LIST
ARCHIVE_URL = "https://snapshot.debian.org/archive/debian-ports/20230724T141507Z/"
APT_SOURCES_LIST = [("sid", ["main"])]
if args.command == "build":
build_sysroot(args.architecture)
elif args.command == "upload":

6
src/build/linux/sysroot_scripts/sysroots.json
View File

@ -34,5 +34,11 @@
"SysrootDir": "debian_bullseye_mipsel-sysroot",
"Tarball": "debian_bullseye_mipsel_sysroot.tar.xz",
"URL": "https://commondatastorage.googleapis.com/chrome-linux-sysroot"
},
"bullseye_riscv64": {
"Key": "",
"Sha1Sum": "",
"SysrootDir": "debian_bullseye_riscv64-sysroot",
"Tarball": "debian_bullseye_riscv64_sysroot.tar.xz"
}
}