Bazel build & Continuous Integration

.gitignore

@@ -1,6 +1,35 @@
+# Ignore backup files.
+*~
+
+# Ignore Vim swap files.
+.*.swp
+
+# Ignore files generated by IDEs.
+/.classpath
+/.factorypath
+/.idea/
+/.project
+/.settings
+/.vscode/
+/bazel.iml
+
+# Ignore all bazel-* symlinks. There is no full list since this can change
+# based on the name of the directory bazel is cloned into.
+/bazel-*
+
+# Ignore outputs generated during Bazel bootstrapping.
+/output/
+
+# Dependency Analysis
+graph.in
+graph.png
+
+# R Artifacts
 .RData
 .Rhistory
 .Rproj.user
 *.Rproj
-
-*.orig
+pkg/rhdfs.bin.Rcheck
+*.log
+pkg/..Rcheck
+pkg.Rcheck

.travis.yml

@@ -0,0 +1,27 @@
+dist: trusty
+
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - wget
+      - pkg-config
+
+before_install:
+  - wget https://github.com/bazelbuild/bazel/releases/download/0.11.0/bazel_0.11.0-linux-x86_64.deb
+  - sudo dpkg -i bazel_0.11.0-linux-x86_64.deb
+  - sudo sh -c 'echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list'
+  - gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
+  - gpg -a --export E084DAB9 | sudo apt-key add -
+  - wget http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.13.0.tar.gz
+  - sudo tar xvf hadoop-2.6.0-cdh5.13.0.tar.gz -C /opt
+  - sudo apt-get update
+  - sudo apt-get -y install r-base
+  - sudo R CMD javareconf
+  - sudo R --quiet -e 'install.packages("rJava", type="source", repos="http://cran.us.r-project.org")'
+  - sudo apt-get install -y texlive-base texlive-extra-utils texlive-latex-base texlive-latex-recommended texlive-generic-recommended
+env:
+  - HADOOP_CMD="/opt/hadoop-2.6.0-cdh5.13.0/bin/hadoop"
+script:
+  - bazel build //...

README.md

@@ -1,4 +1,44 @@
-rhdfs
-=====
+# rhdfs
 
-A package that allows R developers to use Hadoop HDFS, developed as part of the RHadoop project. Please see the [RHadoop wiki](https://github.com/RevolutionAnalytics/RHadoop/wiki) for information. 
+This repository provides an R package for interacting with Hadoop Distributed File System (HDFS).
+
+[![Build Status](https://travis-ci.org/bowlofstew/rhdfs.svg?branch=master)](https://github.com/bowlofstew/rhdfs)
+
+## Compilation Requirements
+
+The basic project requirements are as follows:
+
+* [JDK](https://java.com/en/download/)
+
+* [Bazel](https://bazel.build/)
+
+* [R](https://www.r-project.org/)
+
+## Compilation
+
+In order to compile the package, execute the command, `bazel build //...` and this 
+will build the R package and its Java dependency.
+
+## Publishing to CRAN
+
+1. Execute the command, `bazel build //..`.
+
+2. From the top level project directory, execute the command, `R CMD check --as-cran pkg/`
+
+3. In a web browser, navigate to `http://cran.r-project.org/submit.html` and create a submission for your package.
+
+## Dependency Analysis
+
+In order to perform a dependency analysis of the project, you will need to meet an extra requirement of:
+
+* [GraphViz Dot Compiler](https://www.graphviz.org/)
+
+Once the dependency is satisified, you can execute the following commands to create a depedency graph:
+
+    bazel query 'deps(//pkg:rhdfs)' --output graph > graph.in
+
+    dot -Tpng < graph.in > graph.png
+
+## Author(s)
+
+Stewart Henderson <stewart.henderson@protonmail.com>

WORKSPACE

@@ -0,0 +1,50 @@
+workspace(name = "rhdfs")
+
+# Bazel Rules
+RULES_R_COMMIT="6c6a0536c829e59c1ddcb3a96df0376342f66bf9"
+
+git_repository(
+    name = "com_grail_rules_r",
+    remote = "https://github.com/grailbio/rules_r.git",
+    commit = RULES_R_COMMIT,  
+)
+
+# Java, Maven Dependencies
+maven_jar(
+    name = "org_apache_hadoop_hadoop_core",
+    artifact = "org.apache.hadoop:hadoop-core:1.2.1",
+)
+
+# R Dependencies
+CRAN_BASE_URL="https://cloud.r-project.org/src/contrib"
+CRAN_PACKAGE="tar.gz"
+RJAVA_VERSION="0.9-9"
+DBI_VERSION="0.8"
+BIGLM_VERSION="0.9-1"
+
+new_http_archive(
+    name = "R_rJava",
+    build_file = "cran/BUILD.rJava",
+    strip_prefix = "rJava",
+    urls = [
+        "{}/rJava_{}.{}".format(CRAN_BASE_URL, RJAVA_VERSION, CRAN_PACKAGE)
+    ],
+)
+
+new_http_archive(
+    name = "R_DBI",
+    build_file = "cran/BUILD.DBI",
+    strip_prefix = "DBI",
+    urls = [
+        "{}/DBI_{}.{}".format(CRAN_BASE_URL, DBI_VERSION, CRAN_PACKAGE)
+    ],
+)
+
+new_http_archive(
+    name = "R_biglm",
+    build_file = "cran/BUILD.biglm",
+    strip_prefix = "biglm",
+    urls = [
+        "{}/biglm_{}.{}".format(CRAN_BASE_URL, BIGLM_VERSION, CRAN_PACKAGE)
+    ],
+)

cran/BUILD.DBI

@@ -0,0 +1,15 @@
+# https://cran.r-project.org/web/packages/DBI/index.html
+
+load("@com_grail_rules_r//R:defs.bzl", "r_pkg")
+
+package(default_visibility = ["//visibility:public"])
+
+r_pkg(
+    name = "DBI",
+    srcs = glob(
+        ["**"],
+        exclude = [],
+    ),
+    lazy_data = True,
+    deps = [],
+)

cran/BUILD.biglm

@@ -0,0 +1,17 @@
+# https://cran.r-project.org/web/packages/biglm/index.html
+
+load("@com_grail_rules_r//R:defs.bzl", "r_pkg")
+
+package(default_visibility = ["//visibility:public"])
+
+r_pkg(
+    name = "biglm",
+    srcs = glob(
+        ["**"],
+        exclude = [],
+    ),
+    lazy_data = True,
+    deps = [
+        "@R_DBI//:DBI",
+    ],
+)

cran/BUILD.rJava

@@ -0,0 +1,15 @@
+# https://cran.r-project.org/web/packages/rJava/index.html
+
+load("@com_grail_rules_r//R:defs.bzl", "r_pkg")
+
+package(default_visibility = ["//visibility:public"])
+
+r_pkg(
+    name = "rJava",
+    srcs = glob(
+        ["**"],
+        exclude = [],
+    ),
+    lazy_data = True,
+    deps = [],
+)

java/com/revolutionanalytics/hadoop/hdfs/BUILD

@@ -0,0 +1,16 @@
+package(default_visibility = ["//visibility:public"])
+
+SOURCES = glob(
+    ["**"],
+    exclude = [
+        "BUILD",
+    ],
+)
+
+java_library(
+    name = "rhdfs_lib",
+    srcs = SOURCES,
+    deps = [
+        "@org_apache_hadoop_hadoop_core//jar"
+    ],
+)

pkg/BUILD

@@ -0,0 +1,25 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@com_grail_rules_r//R:defs.bzl", "r_package")
+
+r_package(
+        pkg_name = "rhdfs", 
+        pkg_srcs = glob(
+            ["**"],
+            exclude = [
+                "BUILD",
+                "README.md",
+                ".gitignore",
+                ".Rbuildignore",
+                "rplatform.Rproj",
+                ".Rproject.user"
+            ],
+        ), 
+        pkg_deps = [
+            "@R_rJava//:rJava",
+            "@R_biglm//:biglm",
+        ],
+        pkg_suggested_deps = [
+            "//java/com/revolutionanalytics/hadoop/hdfs:rhdfs_lib"
+        ]
+)

pkg/DESCRIPTION

@@ -1,10 +1,15 @@
 Package: rhdfs 
 Type: Package
 Title: R and Hadoop Distributed Filesystem
-Version: 1.0.8
-Date: 2013-06-20
+Version: 1.0.9
+Date: 2018-03-05
 Author: Revolution Analytics
-Depends: R (>= 2.6.0), methods, rJava (>= 0.8)
-Maintainer: Revolution Analytics <rhadoop@revolutionanalytics.com>
+Depends: 
+    R (>= 3.3.3),
+    methods,
+    rJava (>= 0.9-9)
+SystemRequirements: Java (>= 1.6)
+Maintainer: Stewart Henderson <stewart.henderson@protonmail.com>
 Description: Functions to browse, open, write files to the HDFS
 License: Apache License (== 2.0)
+RoxygenNote: 6.0.1.9000

pkg/NAMESPACE

@@ -5,5 +5,4 @@ importFrom("rJava", ".jclassLoader")
 importFrom("rJava", "J") 
 exportPattern("^hdfs")
 S3method("print", "hdfsFH")
-S3method("as.character", "hdfsFH")
-
+S3method("as.character", "hdfsFH")

pkg/R/zzz.R

@@ -15,10 +15,19 @@
 
 library(utils)
 .hdfsEnv <- new.env()
-.onLoad <- function(libname,pkgname){
+.onLoad <- function(libname,pkgname) {
   vrs <- packageDescription(pkgname, lib.loc = libname, fields = "Version",
                             drop = TRUE)
-  if (Sys.getenv("HADOOP_CMD") == "") stop(sprintf("Environment variable HADOOP_CMD must be set before loading package %s", pkgname))
+  if (Sys.getenv("HADOOP_CMD") == "") {
+    HADOOP_VERSION = "2.6.0"
+    CLOUDERA_VERSION = "5.13.0"
+    CLOUDERA_HADOOP_INSTALLATION_DIR = sprintf("/opt/hadoop-%s-cdh%s", HADOOP_VERSION, CLOUDERA_VERSION)
+    if (!dir.exists(CLOUDERA_HADOOP_INSTALLATION_DIR)) {
+      Sys.setenv(HADOOP_CMD = sprintf("/opt/hadoop-%s-cdh%s/bin/hadoop", HADOOP_VERSION, CLOUDERA_VERSION))
+    }
+  } else {
+    stop(sprintf("Environment variable HADOOP_CMD must be set before loading package %s", pkgname))
+  }
   packageStartupMessage("\nHADOOP_CMD=", Sys.getenv("HADOOP_CMD"))
   packageStartupMessage("\nBe sure to run hdfs.init()")
   #hdfs.init()