Hadoop native libraries not found on OS/X

To get this working on a fresh install of macOS 10.12, I had to do the following:

1. Install build dependencies using homebrew:

   brew install cmake maven openssl protobuf@2.5 snappy

2. Check out hadoop source code

   git clone https://github.com/apache/hadoop.gitcd hadoopgit checkout rel/release-2.7.3

3. Apply the below patch to the build:

   diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txtindex 942b19c..8b34881 100644--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt@@ -16,6 +16,8 @@ # limitations under the License. #+SET(CUSTOM_OPENSSL_PREFIX /usr/local/opt/openssl)+ cmake_minimum_required(VERSION 2.6 FATAL_ERROR) # Default to release builds@@ -116,8 +118,8 @@ set(T main/native/src/test/org/apache/hadoop) GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME) SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})-set_find_shared_library_version("1")-find_package(BZip2 QUIET)+set_find_shared_library_version("1.0")+find_package(BZip2 REQUIRED) if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)     GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)     set(BZIP2_SOURCE_FILESdiff --git a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xmlindex d2ddf89..ac8e351 100644--- a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml+++ b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml@@ -17,4 +17,8 @@ <!-- Put site-specific property overrides in this file. --> <configuration>+<property>+<name>io.compression.codec.bzip2.library</name>+<value>libbz2.dylib</value>+</property> </configuration>diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xmlindex 34c0110..70f23a4 100644--- a/hadoop-tools/hadoop-pipes/pom.xml+++ b/hadoop-tools/hadoop-pipes/pom.xml@@ -52,7 +52,7 @@                     <mkdir dir="${project.build.directory}/native"/>                     <exec executable="cmake" dir="${project.build.directory}/native"                          failonerror="true">-                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>+                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"/>                     </exec>                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true">                       <arg line="VERBOSE=1"/>

4. Build hadoop from source:

   mvn package -Pdist,native -DskipTests -Dtar -Dmaven.javadoc.skip=true

5. Specify JAVA_LIBRARY_PATH when running hadoop:

   $ JAVA_LIBRARY_PATH=/usr/local/opt/openssl/lib:/opt/local/lib:/usr/lib hadoop-dist/target/hadoop-2.7.3/bin/hadoop checknative -a16/10/14 20:16:32 INFO bzip2.Bzip2Factory: Successfully loaded & initialized native-bzip2 library libbz2.dylib16/10/14 20:16:32 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib libraryNative library checking:hadoop:  true /Users/admin/Desktop/hadoop/hadoop-dist/target/hadoop-2.7.3/lib/native/libhadoop.dylibzlib:    true /usr/lib/libz.1.dylibsnappy:  true /usr/local/lib/libsnappy.1.dyliblz4:     true revision:99bzip2:   true /usr/lib/libbz2.1.0.dylibopenssl: true /usr/local/opt/openssl/lib/libcrypto.dylib

There are some missing steps in @andrewdotn's response above:

1) For step (3), create the patch by adding the text posted to a text file e.g. "patch.txt", and then execute "git apply patch.txt"

2) In addition to copying the files as directed by javadba, certain applications also require that you set:

export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${HADOOP_HOME}/lib/nativeexport JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:${HADOOP_HOME}/lib/native

The needed step is to copy the *.dylib from the git sources build dir into the $HADOOP_HOME/<common dir>lib dir for your platform . For OS/X installed via brew it is:

cp /git/hadoop/hadoop-dist/target/hadoop-2.7.1/lib/native/ /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/

We can see the required libs there now:

$ll /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/*.dylib-rwxr-xr-x  1 macuser  staff  149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylib-rwxr-xr-x  1 macuser  staff  149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.1.0.0.dylib

And now the hadoop checknative command works:

$hadoop checknative6/06/15 09:10:59 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib libraryNative library checking:hadoop:  true /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylibzlib:    true /usr/lib/libz.1.dylibsnappy:  falselz4:     true revision:99bzip2:   falseopenssl: false build does not support openssl.