#!/bin/bash # # Build and runs tests for the protobuf project. We use this script to run # tests on kokoro (Ubuntu and MacOS). It can run locally as well but you # will need to make sure the required compilers/tools are available. # For when some other test needs the C++ main build, including protoc and # libprotobuf. LAST_RELEASED=3.9.0 internal_build_cpp() { if [ -f src/protoc ]; then # Already built. return fi # Initialize any submodules. git submodule update --init --recursive ./autogen.sh ./configure CXXFLAGS="-fPIC -std=c++11" # -fPIC is needed for python cpp test. # See python/setup.py for more details make -j$(nproc) } build_cpp() { internal_build_cpp make check -j$(nproc) || (cat src/test-suite.log; false) cd conformance && make test_cpp && cd .. } build_cpp_tcmalloc() { internal_build_cpp ./configure LIBS=-ltcmalloc && make clean && make \ PTHREAD_CFLAGS='-pthread -DGOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN' \ check cd src PPROF_PATH=/usr/bin/google-pprof HEAPCHECK=strict ./protobuf-test } build_cpp_distcheck() { grep -q -- "-Og" src/Makefile.am && echo "The -Og flag is incompatible with Clang versions older than 4.0." && exit 1 # Initialize any submodules. git submodule update --init --recursive ./autogen.sh ./configure make dist # List all files that should be included in the distribution package. git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\ grep -v ".gitignore" | grep -v "java/lite/proguard.pgcfg" |\ grep -v "python/compatibility_tests" | grep -v "python/docs" | grep -v "python/.repo-metadata.json" |\ grep -v "python/protobuf_distutils" | grep -v "csharp/compatibility_tests" > dist.lst # Unzip the dist tar file. DIST=`ls *.tar.gz` tar -xf $DIST cd ${DIST//.tar.gz} # Check if every file exists in the dist tar file. FILES_MISSING="" for FILE in $(<../dist.lst); do [ -f "$FILE" ] || { echo "$FILE is not found!" FILES_MISSING="$FILE $FILES_MISSING" } done cd .. if [ ! -z "$FILES_MISSING" ]; then echo "Missing files in EXTRA_DIST: $FILES_MISSING" exit 1 fi # Do the regular dist-check for C++. make distcheck -j$(nproc) } build_dist_install() { # Create a symlink pointing to python2 and put it at the beginning of $PATH. # This is necessary because the googletest build system involves a Python # script that is not compatible with Python 3. More recent googletest # versions have fixed this, but they have also removed the autotools build # system support that we rely on. This is a temporary workaround to keep the # googletest build working when the default python binary is Python 3. mkdir tmp || true pushd tmp ln -s /usr/bin/python2 ./python popd PATH=$PWD/tmp:$PATH # Initialize any submodules. git submodule update --init --recursive ./autogen.sh ./configure make dist # Unzip the dist tar file and install it. DIST=`ls *.tar.gz` tar -xf $DIST pushd ${DIST//.tar.gz} ./configure && make check -j4 && make install export LD_LIBRARY_PATH=/usr/local/lib # Try to install Java pushd java use_java jdk11 $MVN install popd # Try to install Python python3 -m venv venv source venv/bin/activate pushd python python3 setup.py clean build sdist pip3 install dist/protobuf-*.tar.gz popd deactivate rm -rf python/venv } build_csharp() { # Required for conformance tests and to regenerate protos. internal_build_cpp NUGET=/usr/local/bin/nuget.exe # Disable some unwanted dotnet options export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true export DOTNET_CLI_TELEMETRY_OPTOUT=true # TODO(jtattermusch): is this still needed with "first time experience" # disabled? # Perform "dotnet new" once to get the setup preprocessing out of the # way. That spews a lot of output (including backspaces) into logs # otherwise, and can cause problems. It doesn't matter if this step # is performed multiple times; it's cheap after the first time anyway. # (It also doesn't matter if it's unnecessary, which it will be on some # systems. It's necessary on Jenkins in order to avoid unprintable # characters appearing in the JUnit output.) mkdir dotnettmp (cd dotnettmp; dotnet new > /dev/null) rm -rf dotnettmp # Check that the protos haven't broken C# codegen. # TODO(jonskeet): Fail if regenerating creates any changes. csharp/generate_protos.sh csharp/buildall.sh cd conformance && make test_csharp && cd .. # Run csharp compatibility test between 3.0.0 and the current version. csharp/compatibility_tests/v3.0.0/test.sh 3.0.0 # Run csharp compatibility test between last released and the current version. csharp/compatibility_tests/v3.0.0/test.sh $LAST_RELEASED } build_golang() { # Go build needs `protoc`. internal_build_cpp # Add protoc to the path so that the examples build finds it. export PATH="`pwd`/src:$PATH" export GOPATH="$HOME/gocode" mkdir -p "$GOPATH/src/github.com/protocolbuffers" mkdir -p "$GOPATH/src/github.com/golang" rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf" rm -f "$GOPATH/src/github.com/golang/protobuf" ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf" export PATH="$GOPATH/bin:$PATH" (cd $GOPATH/src/github.com/golang && git clone https://github.com/golang/protobuf.git && cd protobuf && git checkout v1.3.5) go install github.com/golang/protobuf/protoc-gen-go cd examples && PROTO_PATH="-I../src -I." make gotest && cd .. } use_java() { version=$1 case "$version" in jdk11) export PATH=/usr/lib/jvm/java-11-openjdk-amd64/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 ;; jdk8) export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 ;; jdk7) export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; oracle7) export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; esac MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository MVN="$MVN -e --quiet -Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" which java java -version $MVN -version } # --batch-mode suppresses download progress output that spams the logs. MVN="mvn --batch-mode" internal_build_java() { version=$1 dir=java_$version # Java build needs `protoc`. internal_build_cpp cp -r java $dir cd $dir && $MVN clean # Skip tests here - callers will decide what tests they want to run $MVN install -Dmaven.test.skip=true } build_java() { version=$1 internal_build_java $version # Skip the Kotlin tests on Oracle 7 if [ "$version" == "oracle7" ]; then $MVN test -pl bom,lite,core,util else $MVN test fi cd ../.. } # The conformance tests are hard-coded to work with the $ROOT/java directory. # So this can't run in parallel with two different sets of tests. build_java_with_conformance_tests() { # Java build needs `protoc`. internal_build_cpp # This local installation avoids the problem caused by a new version not yet in Maven Central cd java/bom && $MVN install cd ../.. cd java/core && $MVN test && $MVN install cd ../lite && $MVN test && $MVN install cd ../util && $MVN test && $MVN install && $MVN package assembly:single if [ "$version" == "jdk8" ]; then cd ../kotlin && $MVN test && $MVN install cd ../kotlin-lite && $MVN test && $MVN install fi cd ../.. cd conformance && make test_java && cd .. } build_java_jdk7() { use_java jdk7 build_java_with_conformance_tests } build_java_oracle7() { use_java oracle7 build_java oracle7 } build_java_linkage_monitor() { # Linkage Monitor checks compatibility with other Google libraries # https://github.com/GoogleCloudPlatform/cloud-opensource-java/tree/master/linkage-monitor use_java jdk11 internal_build_cpp # Linkage Monitor uses $HOME/.m2 local repository MVN="mvn -e -B -Dhttps.protocols=TLSv1.2" cd java # Installs the snapshot version locally $MVN install -Dmaven.test.skip=true # Linkage Monitor uses the snapshot versions installed in $HOME/.m2 to verify compatibility JAR=linkage-monitor-latest-all-deps.jar curl -v -O "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/${JAR}" # Fails if there's new linkage errors compared with baseline java -jar $JAR com.google.cloud:libraries-bom } build_objectivec_ios() { # Reused the build script that takes care of configuring and ensuring things # are up to date. The OS X test runs the objc conformance test, so skip it # here. objectivec/DevTools/full_mac_build.sh \ --core-only --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance "$@" } build_objectivec_ios_debug() { build_objectivec_ios --skip-xcode-release } build_objectivec_ios_release() { build_objectivec_ios --skip-xcode-debug } build_objectivec_osx() { # Reused the build script that takes care of configuring and ensuring things # are up to date. objectivec/DevTools/full_mac_build.sh \ --core-only --skip-xcode-ios --skip-xcode-tvos } build_objectivec_tvos() { # Reused the build script that takes care of configuring and ensuring things # are up to date. The OS X test runs the objc conformance test, so skip it # here. objectivec/DevTools/full_mac_build.sh \ --core-only --skip-xcode-ios --skip-xcode-osx --skip-objc-conformance "$@" } build_objectivec_tvos_debug() { build_objectivec_tvos --skip-xcode-release } build_objectivec_tvos_release() { build_objectivec_tvos --skip-xcode-debug } build_objectivec_cocoapods_integration() { objectivec/Tests/CocoaPods/run_tests.sh } build_python() { internal_build_cpp cd python tox --skip-missing-interpreters cd .. } build_python_version() { internal_build_cpp cd python envlist=$1 tox -e $envlist cd .. } build_python37() { build_python_version py37-python } build_python38() { build_python_version py38-python } build_python39() { build_python_version py39-python } build_python310() { build_python_version py310-python } build_python_cpp() { internal_build_cpp export LD_LIBRARY_PATH=../src/.libs # for Linux export DYLD_LIBRARY_PATH=../src/.libs # for OS X cd python tox --skip-missing-interpreters cd .. } build_python_cpp_version() { internal_build_cpp export LD_LIBRARY_PATH=../src/.libs # for Linux export DYLD_LIBRARY_PATH=../src/.libs # for OS X cd python envlist=$1 tox -e $envlist cd .. } build_python37_cpp() { build_python_cpp_version py37-cpp } build_python38_cpp() { build_python_cpp_version py38-cpp } build_python39_cpp() { build_python_cpp_version py39-cpp } build_python310_cpp() { build_python_cpp_version py310-cpp } build_ruby23() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.3.8 && cd .. } build_ruby24() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.4 && cd .. } build_ruby25() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.5.1 && cd .. } build_ruby26() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.6.0 && cd .. } build_ruby27() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.7.0 && cd .. } build_ruby30() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-3.0.2 && cd .. } build_ruby31() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-3.1.0 && cd .. } build_jruby92() { internal_build_cpp # For conformance tests. internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes cd ruby && bash travis-test.sh jruby-9.2.20.1 && cd .. } build_jruby93() { internal_build_cpp # For conformance tests. internal_build_java jdk8 && cd .. # For Maven protobuf jar with local changes cd ruby && bash travis-test.sh jruby-9.3.3.0 && cd .. } build_javascript() { internal_build_cpp NODE_VERSION=node-v12.16.3-darwin-x64 NODE_TGZ="$NODE_VERSION.tar.gz" pushd /tmp curl -OL https://nodejs.org/dist/v12.16.3/$NODE_TGZ tar zxvf $NODE_TGZ export PATH=$PATH:`pwd`/$NODE_VERSION/bin popd cd js && npm install && npm test && cd .. cd conformance && make test_nodejs && cd .. } use_php() { VERSION=$1 export PATH=/usr/local/php-${VERSION}/bin:$PATH internal_build_cpp } build_php() { use_php $1 pushd php rm -rf vendor php -v php -m composer update composer test popd (cd conformance && make test_php) } test_php_c() { pushd php rm -rf vendor php -v php -m composer update composer test_c popd (cd conformance && make test_php_c) } build_php_c() { use_php $1 test_php_c } build_php7.0_mac() { internal_build_cpp # Install PHP curl -s https://php-osx.liip.ch/install.sh | bash -s 7.0 PHP_FOLDER=`find /usr/local -type d -name "php5-7.0*"` # The folder name may change upon time test ! -z "$PHP_FOLDER" export PATH="$PHP_FOLDER/bin:$PATH" # Install Composer wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer chmod a+x /usr/local/bin/composer # Install valgrind echo "#! /bin/bash" > valgrind chmod ug+x valgrind sudo mv valgrind /usr/local/bin/valgrind # Test test_php_c } build_php7.3_mac() { internal_build_cpp # Install PHP # We can't test PHP 7.4 with these binaries yet: # https://github.com/liip/php-osx/issues/276 curl -s https://php-osx.liip.ch/install.sh | bash -s 7.3 PHP_FOLDER=`find /usr/local -type d -name "php5-7.3*"` # The folder name may change upon time test ! -z "$PHP_FOLDER" export PATH="$PHP_FOLDER/bin:$PATH" # Install Composer wget https://getcomposer.org/download/2.0.13/composer.phar --progress=dot:mega -O /usr/local/bin/composer chmod a+x /usr/local/bin/composer # Install valgrind echo "#! /bin/bash" > valgrind chmod ug+x valgrind sudo mv valgrind /usr/local/bin/valgrind # Test test_php_c } build_php_compatibility() { internal_build_cpp php/tests/compatibility_test.sh $LAST_RELEASED } build_php_multirequest() { use_php 7.4 php/tests/multirequest.sh } build_php8.0_all() { build_php 8.0 build_php 8.1 build_php_c 8.0 build_php_c 8.1 } build_php_all_32() { build_php 7.0 build_php 7.1 build_php 7.4 build_php_c 7.0 build_php_c 7.1 build_php_c 7.4 build_php_c 7.1-zts build_php_c 7.2-zts build_php_c 7.5-zts } build_php_all() { build_php_all_32 build_php_multirequest build_php_compatibility } build_benchmark() { use_php 7.2 cd kokoro/linux/benchmark && ./run.sh } # -------- main -------- if [ "$#" -ne 1 ]; then echo " Usage: $0 { cpp | cpp_distcheck | csharp | java_jdk7 | java_oracle7 | java_linkage_monitor | objectivec_ios | objectivec_ios_debug | objectivec_ios_release | objectivec_osx | objectivec_tvos | objectivec_tvos_debug | objectivec_tvos_release | objectivec_cocoapods_integration | python | python_cpp | python_compatibility | ruby23 | ruby24 | ruby25 | ruby26 | ruby27 | ruby30 | ruby31 | jruby92 | jruby93 | ruby_all | php_all | php_all_32 | php7.0_mac | php7.3_mac | dist_install | benchmark) " exit 1 fi set -e # exit immediately on error set -x # display all commands cd $(dirname $0) eval "build_$1"