Add dependencies locally

deps/protobuf/kokoro/release/python/linux/build_artifacts.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+set -ex
+
+# change to repo root
+pushd $(dirname $0)/../../../..
+
+# Create stage dir
+ORIGINAL_DIR=`pwd`
+pushd ..
+cp -R $ORIGINAL_DIR stage
+export STAGE_DIR="`pwd`/stage"
+popd
+
+export REPO_DIR=protobuf
+export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"`
+
+export BUILD_COMMIT=`git rev-parse HEAD`
+export PLAT=x86_64
+export UNICODE_WIDTH=32
+export MACOSX_DEPLOYMENT_TARGET=10.9
+
+rm -rf artifacts/
+rm -rf multibuild/
+mkdir artifacts
+export ARTIFACT_DIR=$(pwd)/artifacts
+
+git clone https://github.com/matthew-brett/multibuild.git
+# Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from
+# silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180).
+# IMPORTANT: always pin multibuild at the same commit for:
+# - linux/build_artifacts.sh
+# - linux/build_artifacts.sh
+# - windows/build_artifacts.bat
+(cd multibuild; git checkout b89bb903e94308be79abefa4f436bf123ebb1313)
+cp kokoro/release/python/linux/config.sh config.sh
+
+build_artifact_version() {
+  MB_PYTHON_VERSION=$1
+  cp -R $STAGE_DIR $REPO_DIR
+
+  source multibuild/common_utils.sh
+  source multibuild/travis_steps.sh
+  before_install
+
+  clean_code $REPO_DIR $BUILD_COMMIT
+
+  build_wheel $REPO_DIR/python $PLAT
+
+  mv wheelhouse/* $ARTIFACT_DIR
+
+  # Clean up env
+  rm -rf venv
+  sudo rm -rf $REPO_DIR
+}
+
+build_x86_64_manylinux1_artifact_version() {
+  # Explicitly request building manylinux1 wheels, which is no longer the default.
+  # https://github.com/protocolbuffers/protobuf/issues/9180
+  MB_ML_VER=1
+  build_artifact_version $@
+}
+
+build_x86_64_manylinux2010_artifact_version() {
+  # Explicitly request building manylinux2010 wheels
+  MB_ML_VER=2010
+  build_artifact_version $@
+}
+
+build_crosscompiled_aarch64_manylinux2014_artifact_version() {
+  # crosscompilation is only supported with the dockcross manylinux2014 image
+  DOCKER_IMAGE=dockcross/manylinux2014-aarch64:20210706-65bf2dd
+  MB_ML_VER=2014
+  PLAT=aarch64
+
+  # TODO(jtatermusch): currently when crosscompiling, "auditwheel repair" will be disabled
+  # since auditwheel doesn't work for crosscomiled wheels.
+  build_artifact_version $@
+}
+
+build_x86_64_manylinux1_artifact_version 3.6
+build_x86_64_manylinux1_artifact_version 3.7
+build_x86_64_manylinux1_artifact_version 3.8
+build_x86_64_manylinux1_artifact_version 3.9
+build_x86_64_manylinux2010_artifact_version 3.10
+
+build_crosscompiled_aarch64_manylinux2014_artifact_version 3.7
+build_crosscompiled_aarch64_manylinux2014_artifact_version 3.8
+build_crosscompiled_aarch64_manylinux2014_artifact_version 3.9
+build_crosscompiled_aarch64_manylinux2014_artifact_version 3.10

deps/protobuf/kokoro/release/python/linux/config.sh

@@ -0,0 +1,93 @@
+# Define custom utilities
+# Test for OSX with [ -n "$IS_OSX" ]
+
+function pre_build {
+    # Any stuff that you need to do before you start building the wheels
+    # Runs in the root directory of this repository.
+    pushd protobuf
+
+    if [ "$PLAT" == "aarch64" ]
+    then
+      local configure_host_flag="--host=aarch64"
+    fi
+
+    # Build protoc and libprotobuf
+    ./autogen.sh
+    CXXFLAGS="-fPIC -g -O2" ./configure $configure_host_flag
+    make -j8
+
+    if [ "$PLAT" == "aarch64" ]
+    then
+      # we are crosscompiling for aarch64 while running on x64
+      # the simplest way for build_py command to be able to generate
+      # the protos is by running the protoc process under
+      # an emulator. That way we don't have to build a x64 version
+      # of protoc. The qemu-arm emulator is already included
+      # in the dockcross docker image.
+      # Running protoc under an emulator is fast as protoc doesn't
+      # really do much.
+
+      # create a simple shell wrapper that runs crosscompiled protoc under qemu
+      echo '#!/bin/bash' >protoc_qemu_wrapper.sh
+      echo 'exec qemu-aarch64 "../src/protoc" "$@"' >>protoc_qemu_wrapper.sh
+      chmod ugo+x protoc_qemu_wrapper.sh
+
+      # PROTOC variable is by build_py step that runs under ./python directory
+      export PROTOC=../protoc_qemu_wrapper.sh
+    fi
+
+    # Generate python dependencies.
+    pushd python
+    python setup.py build_py
+    popd
+
+    popd
+}
+
+function bdist_wheel_cmd {
+    # Builds wheel with bdist_wheel, puts into wheelhouse
+    #
+    # It may sometimes be useful to use bdist_wheel for the wheel building
+    # process.  For example, versioneer has problems with versions which are
+    # fixed with bdist_wheel:
+    # https://github.com/warner/python-versioneer/issues/121
+    local abs_wheelhouse=$1
+
+    # Modify build version
+    pwd
+    ls
+
+    if [ "$PLAT" == "aarch64" ]
+    then
+      # when crosscompiling for aarch64, --plat-name needs to be set explicitly
+      # to end up with correctly named wheel file
+      # the value should be manylinuxABC_ARCH and dockcross docker image
+      # conveniently provides the value in the AUDITWHEEL_PLAT env
+      local plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
+
+      # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
+      export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(python -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
+    fi
+
+    python setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag
+    cp dist/*.whl $abs_wheelhouse
+}
+
+function build_wheel {
+    build_wheel_cmd "bdist_wheel_cmd" $@
+}
+
+function run_tests {
+    # Runs tests on installed distribution from an empty directory
+    python --version
+    python -c "from google.protobuf.pyext import _message;"
+}
+
+if [ "$PLAT" == "aarch64" ]
+then
+  # when crosscompiling for aarch64, override the default multibuild's repair_wheelhouse logic
+  # since "auditwheel repair" doesn't work for crosscompiled wheels
+  function repair_wheelhouse {
+      echo "Skipping repair_wheelhouse since auditwheel requires build architecture to match wheel architecture."
+  }
+fi

deps/protobuf/kokoro/release/python/linux/continuous.cfg

@@ -0,0 +1,8 @@
+# Config file for running tests in Kokoro
+build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}

deps/protobuf/kokoro/release/python/linux/presubmit.cfg

@@ -0,0 +1,8 @@
+# Config file for running tests in Kokoro
+build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}

deps/protobuf/kokoro/release/python/linux/release.cfg

@@ -0,0 +1,8 @@
+# Configuration for Linux release builds
+build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}