diff --git a/.gitignore b/.gitignore index a1c2a238a965f004ff76978ac1086aa6fe95caea..36f2b0786fad2032f47b2ece3ec41a932b94258d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,105 @@ -# Compiled class file -*.class +# Byte-compiled / optimized / DLL files +__pycache__ +*.py[cod] +*$py.class -# Log file -*.log +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs*/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +node_modules -# BlueJ files -*.ctxt +# Installer logs +pip-log.txt +pip-delete-this-directory.txt -# Mobile Tools for Java (J2ME) -.mtj.tmp/ +# Unit test / coverage reports +htmlcov/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ -# Package Files # -*.jar -*.war -*.nar -*.ear -*.zip +# Environments +env +env3 +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +.python-version +.DS_Store +.idea +.mvn/ +mvnw +mvnw.cmd +*.iml +*.gradle +*.tgz *.tar.gz -*.rar +*.jar +target +dev-support/mini-submarine/database +dev-support/pysubmarine/sdk +submarine-workbench/workbench-web/node +submarine-workbench/workbench-web/node_modules +submarine-workbench/workbench-web/dist +submarine-workbench/workbench-web/package-lock.json +submarine-workbench/workbench-web/npm-debug.log* +submarine-test/e2e/Driver + +# interpreter temp files +derby.log +submarine-workbench/interpreter/spark-interpreter/metastore_db/ +spark-1.*-bin-hadoop* +.spark-dist + +# submarine-cloud +submarine-cloud/vendor/* +submarine-cloud/output/* +submarine-cloud/hack/conf/* +submarine-cloud/hack/output/* +submarine-cloud/bin/* -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* +submarine-security/spark-security/dependency-reduced-pom.xml +submarine-security/spark-security/derby.log + +# submarine-cloud-v2 +submarine-cloud-v2/vendor/* +submarine-cloud-v2/submarine-operator +submarine-cloud-v2/helm-charts/submarine-operator/charts/* + +# vscode file +.project +.classpath +.settings +.factorypath +.vscode/settings.json +.vscode/tasks.json + +# jupyter notebook checkpoints +.ipynb_checkpoints + +# redundant logging files +*.log diff --git a/LICENSE b/LICENSE index 261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64..400e5543d43de4eb5ac5ba3c3bb9b986ed040a95 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -199,3 +200,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +-------------------------------------------------------------------------------- +This product bundles various third-party source code under other open source +licenses. This section summarizes those components and their licenses. +See licenses/ for text of these licenses. \ No newline at end of file diff --git a/LICENSE-binary b/LICENSE-binary new file mode 100644 index 0000000000000000000000000000000000000000..38dae8af299fb62a1a1442addab5dd25a3dbfb97 --- /dev/null +++ b/LICENSE-binary @@ -0,0 +1,358 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +This project bundles some components that are also licensed under the Apache +License Version 2.0: +cglib:cglib:2.2.2 +com.fasterxml.jackson.core:jackson-annotations:2.7.8 +com.fasterxml.jackson.core:jackson-databind:2.7.8 +com.fasterxml.jackson.core:jackson-core:2.7.8 +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.8.10 +com.fasterxml.woodstox:woodstox-core:5.0.3 +com.github.stephenc.jcip:jcip-annotations:1.0-1 +com.google.code.findbugs:jsr305:3.0.0 +com.google.code.findbugs:jsr305:1.3.9 +com.google.code.gson:gson:2.8.1 +com.google.code.gson:gson:2.2.4 +com.google.guava:guava:11.0.2 +com.google.inject:guice:3 +com.google.inject.extensions:guice-servlet:3 +com.jamesmurty.utils:java-xmlbuilder:0.4 +com.nimbusds:nimbus-jose-jwt:4.41.1 +com.squareup.okhttp:okhttp:2.7.5 +com.squareup.okio:okio:1.6.0 +commons-beanutils:commons-beanutils:1.7.0 +commons-beanutils:commons-beanutils-core:1.8.0 +commons-cli:commons-cli:1.2 +commons-codec:commons-codec:1.4 +commons-collections:commons-collections:3.2.2 +commons-configuration:commons-configuration:1.6 +commons-configuration:commons-configuration:1.1 +commons-daemon:commons-daemon:1.0.13 +commons-digester:commons-digester:1.8 +commons-io:commons-io:2.4 +commons-lang:commons-lang:2.6 +commons-logging:commons-logging:1.1.3 +commons-logging:commons-logging:1.1.1 +commons-net:commons-net:3.1 +io.netty:netty:3.7.0.Final +io.netty:netty-all:4.0.23.Final +javax.inject:javax.inject:1 +javax.validation:validation-api:1.1.0.Final +log4j:log4j:1.2.17 +net.java.dev.jets3t:jets3t:0.9.0 +net.lingala.zip4j:zip4j:1.3.2 +net.minidev:accessors-smart:1.2 +net.minidev:json-smart:2.3 +org.apache.avro:avro:1.7.7 +org.apache.avro:avro:1.8.2 +org.apache.commons:commons-lang3:3.4 +org.apache.commons:commons-lang3:3.7 +org.apache.commons:commons-text:1.4 +org.apache.commons:commons-compress:1.4.1 +org.apache.commons:commons-math3:3.1.1 +org.apache.curator:curator-client:2.7.1 +org.apache.curator:curator-framework:2.7.1 +org.apache.curator:curator-recipes:2.7.1 +org.apache.directory.api:api-asn1-api:1.0.0-M20 +org.apache.directory.api:api-util:1.0.0-M20 +org.apache.directory.server:apacheds-i18n:2.0.0-M15 +org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15 +org.apache.hadoop:hadoop-yarn-api:2.9.2 +org.apache.hadoop:hadoop-yarn-client:2.9.2 +org.apache.hadoop:hadoop-yarn-common:2.9.2 +org.apache.hadoop:hadoop-common:2.9.2 +org.apache.hadoop:hadoop-annotations:2.9.2 +org.apache.hadoop:hadoop-auth:2.9.2 +org.apache.hadoop:hadoop-hdfs:2.9.2 +org.apache.hadoop:hadoop-hdfs-client:2.9.2 +org.apache.htrace:htrace-core4:4.1.0-incubating +org.apache.httpcomponents:httpclient:4.5.2 +org.apache.httpcomponents:httpcore:4.4.4 +org.apache.submarine:submarine-core:0.5.0 +org.apache.zookeeper:zookeeper:3.4.6 +org.codehaus.jackson:jackson-core-asl:1.9.13 +org.codehaus.jackson:jackson-mapper-asl:1.9.13 +org.codehaus.jackson:jackson-jaxrs:1.9.13 +org.codehaus.jackson:jackson-xc:1.9.13 +org.codehaus.jettison:jettison:1.1 +org.eclipse.jetty:jetty-jmx:9.4.18.v20190429 +org.eclipse.jetty:jetty-webapp:9.4.18.v20190429 +org.eclipse.jetty:jetty-client:9.4.18.v20190429 +org.eclipse.jetty:jetty-http:9.4.18.v20190429 +org.eclipse.jetty:jetty-io:9.4.18.v20190429 +org.eclipse.jetty:jetty-security:9.4.18.v20190429 +org.eclipse.jetty:jetty-server:9.4.18.v20190429 +org.eclipse.jetty:jetty-servlet:9.4.18.v20190429 +org.eclipse.jetty:jetty-util:9.4.18.v20190429 +org.eclipse.jetty:jetty-xml:9.4.18.v20190429 +org.eclipse.jetty.websocket:websocket-server:9.4.18.v20190429 +org.eclipse.jetty.websocket:websocket-api:9.4.18.v20190429 +org.eclipse.jetty.websocket:websocket-client:9.4.18.v20190429 +org.eclipse.jetty.websocket:websocket-common:9.4.18.v20190429 +org.eclipse.jetty.websocket:websocket-servlet:9.4.18.v20190429 +org.mortbay.jetty:jetty-sslengine:6.1.26 +org.mortbay.jetty:jetty-util:6.1.26 +org.mortbay.jetty:jetty:6.1.26 +org.mybatis:mybatis:3.2.8 +org.xerial.snappy:snappy-java:1.0.5 +org.xerial.snappy:snappy-java:1.1.1.3 +org.yaml:snakeyaml:1.16 +xerces:xercesImpl:2.9.1 +xml-apis:xml-apis:1.3.04 + + +-------------------------------------------------------------------------------- +This product bundles various third-party binaries under other open source +licenses. This section summarizes those components and their licenses. +See licenses-binary/ for text of these licenses. + +MIT License +----------- +org.slf4j:slf4j-api:1.7.25 +com.github.pagehelper:Mybatis-PageHelper:5.1.10 + + +Eclipse Distribution License - v 1.0 +----------- +org.eclipse.jgit + + +Public Domain +------------- +aopalliance:aopalliance:1.0 + + +BSD 2-Clause +------------ +com.linkedin.tony:tony-core:0.3.21 + + +BSD 3-Clause +------------ +org.hamcrest:hamcrest-core:1.3 +com.google.protobuf:protobuf-java:2.5.0 +com.thoughtworks.paranamer:paranamer:2.3 +org.ow2.asm:asm:5.0.4 +asm:asm:3.3.1 + + +CDDL 1.1 + GPLv2 with classpath exception +------------ +org.glassfish.jersey.core:jersey-common:2.27 +org.glassfish.jersey.media:jersey-media-json-jackson:2.27 +org.glassfish.jersey.containers:jersey-container-servlet-core +org.glassfish.jersey.inject:jersey-hk2:2.27 +org.glassfish.jersey.ext:jersey-entity-filtering:2.27 +com.sun.jersey:jersey-client:1.9 +com.sun.jersey.contribs:jersey-guice:1.9 +com.sun.xml.bind:jaxb-impl:2.2.3-1 +javax.xml.bind:jaxb-api:2.2.11 +javax.ws.rs:javax.ws.rs-api:2.1 +javax.annotation:javax.annotation-api:1.2 +javax.servlet:javax.servlet-api:3.1.0 +org.glassfish.hk2:hk2-api:2.5.0-b42 +org.glassfish.hk2.external:aopalliance-repackaged:2.5.0-b42 +javax.servlet.jsp:jsp-api:2.1 diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..4ec5251bf2f1acfd85ac8cc266a67fe2f1177ce7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,27 @@ +Apache Submarine +Copyright 2019 and onwards The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Export Control Notice +--------------------- + +This distribution includes cryptographic software. The country in +which you currently reside may have restrictions on the import, +possession, use, and/or re-export to another country, of +encryption software. BEFORE using any encryption software, please +check your country's laws, regulations and policies concerning the +import, possession, or use, and re-export of encryption software, to +see if this is permitted. See for more +information. + +The U.S. Government Department of Commerce, Bureau of Industry and +Security (BIS), has classified this software as Export Commodity +Control Number (ECCN) 5D002.C.1, which includes information security +software using or performing cryptographic functions with asymmetric +algorithms. The form and manner of this Apache Software Foundation +distribution makes it eligible for export under the License Exception +ENC Technology Software Unrestricted (TSU) exception (see the BIS +Export Administration Regulations, Section 740.13) for both object +code and source code. \ No newline at end of file diff --git a/NOTICE-binary b/NOTICE-binary new file mode 100644 index 0000000000000000000000000000000000000000..8b0120677dfde41c0cdbfbabb843be354c88eb0d --- /dev/null +++ b/NOTICE-binary @@ -0,0 +1,66 @@ +Apache Submarine +Copyright 2019 and onwards The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Export Control Notice +--------------------- + +This distribution includes cryptographic software. The country in +which you currently reside may have restrictions on the import, +possession, use, and/or re-export to another country, of +encryption software. BEFORE using any encryption software, please +check your country's laws, regulations and policies concerning the +import, possession, or use, and re-export of encryption software, to +see if this is permitted. See for more +information. + +The U.S. Government Department of Commerce, Bureau of Industry and +Security (BIS), has classified this software as Export Commodity +Control Number (ECCN) 5D002.C.1, which includes information security +software using or performing cryptographic functions with asymmetric +algorithms. The form and manner of this Apache Software Foundation +distribution makes it eligible for export under the License Exception +ENC Technology Software Unrestricted (TSU) exception (see the BIS +Export Administration Regulations, Section 740.13) for both object +code and source code. + + +-------------------------------------------------------------------------------- +This product bundles a same/modified portion of third-party binaries +and optionally depends on them (NO NEED FOR PERMISSIVELY-LICENSED DEPENDENCIES, +aka, Category A, +per http://www.apache.org/dev/licensing-howto.html#permissive-deps). The license +text can be found in licenses-binary folder. + +TonY +------------- +Copyright 2018 LinkedIn Corporation +All Rights Reserved. + +Licensed under the BSD 2-Clause License (the "License"). +See LICENSE in the project root for license information. + +================================================================================ + +This product includes/uses TensorFlow (https://github.com/tensorflow) +Copyright 2018 The TensorFlow Authors +License: Apache-2.0 + +This product includes/uses TensorFlowOnYARN (https://github.com/Intel-bigdata/TensorFlowOnYARN) +Copyright 2018 Intel-bigdata +License: Apache-2.0 + +================================================================================ + +In addition, this product automatically loads third party code from an external repository +using the Gradle build system. Such third party code is subject to other license +terms than as set forth above. In addition, such third party code may also +depend on and load multiple tiers of dependencies. + + +jgit +------------- +Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors. +Licenced under Eclipse Distribution License - v 1.0 \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2980443ee5ab5876ae0805b4ba87c3bdde9dab87 --- /dev/null +++ b/README.md @@ -0,0 +1,162 @@ + + + + +![color_logo_with_text](docs/assets/color_logo_with_text.png) + +[![Build Status](https://travis-ci.org/apache/submarine.svg?branch=master)](https://travis-ci.org/apache/submarine) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![HitCount](http://hits.dwyl.io/apache/submarine.svg)](http://hits.dwyl.io/apache/submarine) [![PyPI version](https://badge.fury.io/py/apache-submarine.svg)](https://badge.fury.io/py/apache-submarine) + +# What is Apache Submarine? + +**Apache Submarine** (Submarine for short) is an **End-to-End Machine Learning PLATFORM** to allow data scientists to create end-to-end machine learning workflows. To elaborate, on **Submarine**, data scientists can finish each stage in the ML model lifecycle, including data exploration, data pipeline creation, model training, serving, and monitoring. + +## Why Submarine? + +Some open-source and commercial projects are trying to build an end-to-end ML platform. What's the vision of Submarine? + +### Problems + +1) Many platforms lack easy-to-use user interfaces (API, SDK, and IDE, etc.) +2) In the same company, data scientists in different teams usually spend much time on developments of existing feature sets and models. +3) Data scientists put emphasis on domain-specific tasks (e.g. Click-Through-Rate), but they need to implement their models from scratch with SDKs provided by existing platforms. +4) Many platforms lack a unified workbench to manage each component in the ML lifecycle. + +_Theodore Levitt_ once said: + +``` +“People don’t want to buy a quarter-inch drill. They want a quarter-inch hole.” +``` + +### Goals of Submarine + +#### Model Training (Experiment) +- Run/Track distributed training `experiment` on prem or cloud via easy-to-use UI/API/SDK. +- Easy for data scientists to manage versions of `experiment` and dependencies of `environment` +- Support popular machine learning frameworks, including **TensorFlow**, **PyTorch**, **Horovod**, and **MXNet** +- Provide pre-defined **template** for data scientists to implement domain-specific tasks easily (e.g. using DeepFM template to build a CTR prediction model) +- Support many compute resources (e.g. CPU and GPU, etc.) +- Support **Kubernetes** and **YARN** +- Pipeline is also on the backlog, we will look into pipeline for training in the future. + +#### Notebook Service + +- Submarine aims to provide a notebook service (e.g. Jupyter notebook) which allows users to manage notebook instances running on the cluster. + +#### Model Management (Serving/versioning/monitoring, etc.) + +- Model management for model-serving/versioning/monitoring is on the roadmap. + +## Easy-to-use User Interface + +As mentioned above, Submarine attempts to provide **Data-Scientist-friendly** UI to make data scientists have a good user experience. Here're some examples. + +### Example: Submit a distributed Tensorflow experiment via Submarine Python SDK + +#### Run a Tensorflow Mnist experiment +```python + +# New a submarine client of the submarine server +submarine_client = submarine.ExperimentClient(host='http://localhost:8080') + +# The experiment's environment, could be Docker image or Conda environment based +environment = EnvironmentSpec(image='apache/submarine:tf-dist-mnist-test-1.0') + +# Specify the experiment's name, framework it's using, namespace it will run in, +# the entry point. It can also accept environment variables. etc. +# For PyTorch job, the framework should be 'Pytorch'. +experiment_meta = ExperimentMeta(name='mnist-dist', + namespace='default', + framework='Tensorflow', + cmd='python /var/tf_dist_mnist/dist_mnist.py --train_steps=100') +# 1 PS task of 2 cpu, 1GB +ps_spec = ExperimentTaskSpec(resources='cpu=2,memory=1024M', + replicas=1) +# 1 Worker task +worker_spec = ExperimentTaskSpec(resources='cpu=2,memory=1024M', + replicas=1) + +# Wrap up the meta, environment and task specs into an experiment. +# For PyTorch job, the specs would be "Master" and "Worker". +experiment_spec = ExperimentSpec(meta=experiment_meta, + environment=environment, + spec={'Ps':ps_spec, 'Worker': worker_spec}) + +# Submit the experiment to submarine server +experiment = submarine_client.create_experiment(experiment_spec=experiment_spec) + +# Get the experiment ID +id = experiment['experimentId'] + +``` + +#### Query a specific experiment +```python +submarine_client.get_experiment(id) +``` + +#### Wait for finish + +```python +submarine_client.wait_for_finish(id) +``` + +#### Get the experiment's log +```python +submarine_client.get_log(id) +``` + +#### Get all running experiment +```python +submarine_client.list_experiments(status='running') +``` + +For a quick-start, see [Submarine On K8s](docs/userdocs/k8s/README.md) + + +### Example: Submit a pre-defined experiment template job + +### Example: Submit an experiment via Submarine UI + +(Available on 0.6.0, see Roadmap) + +## Architecture, Design and requirements + +If you want to know more about Submarine's architecture, components, requirements and design doc, they can be found on [Architecture-and-requirement](docs/design/architecture-and-requirements.md) + +Detailed design documentation, implementation notes can be found at: [Implementation notes](docs/design/implementation-notes.md) + +## Apache Submarine Community + +Read the [Apache Submarine Community Guide](./docs/community/README.md) + +How to contribute [Contributing Guide](./docs/community/contributing.md) + +Issue Tracking: https://issues.apache.org/jira/projects/SUBMARINE + +## User Document + +See [User Guide Home Page](docs/user-guide-home.md) + +## Developer Document + +See [Developer Guide Home Page](docs/development-guide-home.md) + +## Roadmap + +What to know more about what's coming for Submarine? Please check the roadmap out: https://cwiki.apache.org/confluence/display/SUBMARINE/Roadmap + +## License + +The Apache Submarine project is licensed under the Apache 2.0 License. See the [LICENSE](./LICENSE) file for details. diff --git a/bin/common.sh b/bin/common.sh new file mode 100755 index 0000000000000000000000000000000000000000..ecf946995d8a400adeb4abf29bf968c44f00c14d --- /dev/null +++ b/bin/common.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export DEFAULT_MYSQL_VERSION=5.1.39 + +if [[ -L ${BASH_SOURCE-$0} ]]; then + FWDIR=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + FWDIR=$(dirname "${BASH_SOURCE-$0}") +fi + +if [[ -z "${SUBMARINE_HOME}" ]]; then + # Make SUBMARINE_HOME look cleaner in logs by getting rid of the + # extra ../ + export SUBMARINE_HOME="$(cd "${FWDIR}/.."; pwd)" +fi + +if [[ -z "${SUBMARINE_CONF_DIR}" ]]; then + export SUBMARINE_CONF_DIR="${SUBMARINE_HOME}/conf" +fi + +if [[ -z "${SUBMARINE_LOG_DIR}" ]]; then + export SUBMARINE_LOG_DIR="${SUBMARINE_HOME}/logs" +fi + +if [[ -f "${SUBMARINE_CONF_DIR}/submarine-env.sh" ]]; then + . "${SUBMARINE_CONF_DIR}/submarine-env.sh" +fi + +SUBMARINE_SERVER_CLASSPATH+=":${SUBMARINE_CONF_DIR}" + +function add_each_jar_in_dir(){ + if [[ -d "${1}" ]]; then + for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do + SUBMARINE_SERVER_CLASSPATH="$jar:$SUBMARINE_SERVER_CLASSPATH" + done + fi +} + +function add_each_jar_in_dir_recursive(){ + if [[ -d "${1}" ]]; then + for jar in $(find -L "${1}" -type f -name '*jar'); do + SUBMARINE_SERVER_CLASSPATH="$jar:$SUBMARINE_SERVER_CLASSPATH" + done + fi +} + +function add_jar_in_dir(){ + if [[ -d "${1}" ]]; then + SUBMARINE_SERVER_CLASSPATH="${1}/*:${SUBMARINE_SERVER_CLASSPATH}" + fi +} + +function download_mysql_jdbc_jar(){ + if [[ -z "${MYSQL_JAR_URL}" ]]; then + if [[ -z "${MYSQL_VERSION}" ]]; then + MYSQL_VERSION="${DEFAULT_MYSQL_VERSION}" + fi + MYSQL_JAR_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_VERSION}/mysql-connector-java-${MYSQL_VERSION}.jar" + fi + + echo "Downloading mysql jdbc jar from ${MYSQL_JAR_URL}." + if type wget >/dev/null 2>&1; then + wget ${MYSQL_JAR_URL} -P "${SUBMARINE_HOME}/lib" --no-check-certificate + elif type curl >/dev/null 2>&1; then + curl -o "${SUBMARINE_HOME}/lib/mysql-connector-java-${MYSQL_VERSION}.jar" ${MYSQL_JAR_URL} + else + echo 'We need a tool to transfer data from or to a server. Such as wget/curl.' + echo 'Bye, bye!' + exit 1 + fi + + echo "Mysql jdbc jar is downloaded and put in the path of submarine/lib." +} + +JAVA_OPTS+=" -Dfile.encoding=UTF-8" +JAVA_OPTS+=" -Dlog4j.configuration=file://${SUBMARINE_CONF_DIR}/log4j.properties" +export JAVA_OPTS + +if [[ -n "${JAVA_HOME}" ]]; then + JAVA_RUNNER="${JAVA_HOME}/bin/java" +else + JAVA_RUNNER=java +fi +export JAVA_RUNNER diff --git a/bin/submarine-daemon.sh b/bin/submarine-daemon.sh new file mode 100755 index 0000000000000000000000000000000000000000..f7fcdc8d365a5cf9946827c8e7601abd4ae5cb44 --- /dev/null +++ b/bin/submarine-daemon.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# description: Start and stop daemon script for. +# + +USAGE="-e Usage: submarine-daemon.sh {start|stop|restart|status}" + +if [ -L ${BASH_SOURCE-$0} ]; then + BIN=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + BIN=$(dirname ${BASH_SOURCE-$0}) +fi +export BIN=$(cd "${BIN}">/dev/null; pwd) +GET_MYSQL_JAR=false + +. "${BIN}/common.sh" + +cd ${BIN}/>/dev/null + +SUBMARINE_SERVER_NAME="Submarine Server" +SUBMARINE_SERVER_LOGFILE="${SUBMARINE_LOG_DIR}/submarine.log" +SUBMARINE_SERVER_MAIN=org.apache.submarine.server.SubmarineServer +JAVA_OPTS+="${SUBMARINE_SERVER_JAVA_OPTS} ${SUBMARINE_SERVER_MEM} -Dsubmarine.log.file=${SUBMARINE_SERVER_LOGFILE}" + +add_jar_in_dir "${BIN}/../lib" + +function initialize_default_directories() { + if [[ ! -d "${SUBMARINE_LOG_DIR}" ]]; then + echo "Log dir doesn't exist, create ${SUBMARINE_LOG_DIR}" + $(mkdir -p "${SUBMARINE_LOG_DIR}") + fi +} + +function found_submarine_server_pid() { + process='SubmarineServer'; + RUNNING_PIDS=$(ps x | grep ${process} | grep -v grep | awk '{print $1}'); + + if [[ -z "${RUNNING_PIDS}" ]]; then + return + fi + + if ! kill -0 ${RUNNING_PIDS} > /dev/null 2>&1; then + echo "${SUBMARINE_SERVER_NAME} running but process is dead" + fi + + echo "${RUNNING_PIDS}" +} + +function wait_for_submarine_server_to_die() { + local pid + local count + + pid=`found_submarine_server_pid` + timeout=10 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +function check_jdbc_jar() { + if [[ -d "${1}" ]]; then + mysql_connector_exists=$(find -L "${1}" -name "mysql-connector*") + if [[ -z "${mysql_connector_exists}" ]]; then + if [[ ${GET_MYSQL_JAR} = true ]]; then + download_mysql_jdbc_jar + else + echo -e "\\033[31mError: There is no mysql jdbc jar in lib.\\033[0m" + echo -e "\\033[31mPlease download a mysql jdbc jar and put it under lib manually.\\033[0m" + echo -e "\\033[31mOr add a parameter getMysqlJar, like this:\n./bin/submarine-daemon.sh start getMysqlJar\\033[0m" + echo -e "\\033[31mIt would download mysql jdbc jar automatically.\\033[0m" + exit 1 + fi + fi + fi +} + +function start() { + local pid + + pid=`found_submarine_server_pid` + if [[ ! -z "$pid" && "$pid" != 0 ]]; then + echo "${SUBMARINE_SERVER_NAME}:${pid} is already running" + return 1; + fi + + check_jdbc_jar "${BIN}/../lib" + + initialize_default_directories + + echo "SUBMARINE_SERVER_CLASSPATH: ${SUBMARINE_SERVER_CLASSPATH}" >> "${SUBMARINE_SERVER_LOGFILE}" + + nohup $JAVA_RUNNER $JAVA_OPTS -cp $SUBMARINE_SERVER_CLASSPATH $SUBMARINE_SERVER_MAIN >> "${SUBMARINE_SERVER_LOGFILE}" 2>&1 < /dev/null & + pid=$! + if [[ ! -z "${pid}" ]]; then + echo "${SUBMARINE_SERVER_NAME} start" + return 0; + fi +} + +function stop() { + local pid + pid=`found_submarine_server_pid` + + if [[ -z "$pid" ]]; then + echo "${SUBMARINE_SERVER_NAME} is not running" + return 0; + else + # submarine workbench daemon kill + wait_for_submarine_server_to_die + echo "${SUBMARINE_SERVER_NAME} stop" + fi +} + +function find_submarine_server_process() { + local pid + pid=`found_submarine_server_pid` + + if [[ -z "$pid" ]]; then + echo "${SUBMARINE_SERVER_NAME} is not running" + return 1 + else + if ! kill -0 ${pid} > /dev/null 2>&1; then + echo "${SUBMARINE_SERVER_NAME} running but process is dead" + return 1 + else + echo "${SUBMARINE_SERVER_NAME} is running" + fi + fi +} + +if [[ "$2" = "getMysqlJar" ]]; then + export GET_MYSQL_JAR=true +fi + +case "${1}" in + start) + start + ;; + stop) + stop + ;; + restart) + echo "${SUBMARINE_SERVER_NAME} is restarting" >> "${SUBMARINE_SERVER_LOGFILE}" + stop + start + ;; + status) + find_submarine_server_process + ;; + *) + echo ${USAGE} +esac diff --git a/bin/submarine.sh b/bin/submarine.sh new file mode 100755 index 0000000000000000000000000000000000000000..8a5d6e8d72157e00bc3ef0ec56065ef3621e2d51 --- /dev/null +++ b/bin/submarine.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +USAGE="Usage: bin/submarine.sh [--config ]" + +if [[ "$1" == "--config" ]]; then + shift + conf_dir="$1" + if [[ ! -d "${conf_dir}" ]]; then + echo "ERROR : ${conf_dir} is not a directory" + echo ${USAGE} + exit 1 + else + export SUBMARINE_CONF_DIR="${conf_dir}" + fi + shift +fi + +if [ -L ${BASH_SOURCE-$0} ]; then + BIN=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + BIN=$(dirname ${BASH_SOURCE-$0}) +fi +export BIN=$(cd "${BIN}">/dev/null; pwd) +GET_MYSQL_JAR=false + +. "${BIN}/common.sh" + +cd ${BIN}/>/dev/null + +SUBMARINE_SERVER_NAME="Submarine Server" +SUBMARINE_SERVER_LOGFILE="${SUBMARINE_LOG_DIR}/submarine.log" +SUBMARINE_SERVER_MAIN=org.apache.submarine.server.SubmarineServer +JAVA_OPTS+="${SUBMARINE_SERVER_JAVA_OPTS:-""} ${SUBMARINE_SERVER_MEM:-""} -Dsubmarine.log.file=${SUBMARINE_SERVER_LOGFILE}" + +add_jar_in_dir "${BIN}/../lib" + +if [[ ! -d "${SUBMARINE_LOG_DIR}" ]]; then + echo "Log dir doesn't exist, create ${SUBMARINE_LOG_DIR}" + $(mkdir -p "${SUBMARINE_LOG_DIR}") +fi + +exec $JAVA_RUNNER $JAVA_OPTS -cp ${SUBMARINE_SERVER_CLASSPATH} ${SUBMARINE_SERVER_MAIN} "$@" >> "${SUBMARINE_SERVER_LOGFILE}" 2>&1 diff --git a/conf/log4j.properties b/conf/log4j.properties new file mode 100644 index 0000000000000000000000000000000000000000..2f44c7217d92b7d27c10e54c45cce707a572fb92 --- /dev/null +++ b/conf/log4j.properties @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +submarine.log.threshold=ALL +submarine.root.logger=INFO, DRFA, console +submarine.log.dir=/tmp +submarine.log.file=submarine.log + +log4j.rootLogger=${submarine.root.logger}, EventCounter + +# Logging Threshold +log4j.threshold=${submarine.log.threshold} + +# +# Daily Rolling File Appender +# +# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files +# for different CLI session. + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + +log4j.appender.DRFA.File=${submarine.log.dir}/${submarine.log.file} + +# Rollover at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 diff --git a/conf/log4j.properties.template b/conf/log4j.properties.template new file mode 100755 index 0000000000000000000000000000000000000000..579a382764c700e706f0e56ea0c1e1083bad77ce --- /dev/null +++ b/conf/log4j.properties.template @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +submarine.log.threshold=ALL +submarine.root.logger=INFO,DRFA,console +submarine.log.dir=/tmp +submarine.log.file=submarine.log + +log4j.rootLogger=${submarine.root.logger}, EventCounter + +# Logging Threshold +log4j.threshold=${submarine.log.threshold} + +# +# Daily Rolling File Appender +# +# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files +# for different CLI session. + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + +log4j.appender.DRFA.File=${submarine.log.dir}/${submarine.log.file} + +# Rollover at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 diff --git a/conf/submarine-env.sh.template b/conf/submarine-env.sh.template new file mode 100644 index 0000000000000000000000000000000000000000..97ae1d860b01b4f943582ec3b5b5a8094acb43a7 --- /dev/null +++ b/conf/submarine-env.sh.template @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# export JAVA_HOME=java + +# Debug Submarine server +# export SUBMARINE_SERVER_JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000" + +# Set Submarine server memory +# export SUBMARINE_SERVER_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m" + +# Set Submarine server classpath. If you want to visit hdfs, just add hadoop +# configuration path. +# export SUBMARINE_SERVER_CLASSPATH+=":/usr/local/hadoop/etc/hadoop" diff --git a/conf/submarine-site.xml b/conf/submarine-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..eba252937c94678050fe322a719cd81f1ae154d1 --- /dev/null +++ b/conf/submarine-site.xml @@ -0,0 +1,164 @@ + + + + + + + + submarine.cluster.addr + + submarine cluster address list, e.g. ip1:port1;ip2:port2;ip3:port3 + + + + submarine.server.addr + 0.0.0.0 + Server address + + + + submarine.server.port + 8080 + Server port. + + + + submarine.server.ssl + false + Should SSL be used by the submarine server? + + + + submarine.server.ssl.port + 8443 + Server ssl port. (used when ssl property is set to true) + + + + submarine.server.ssl.client.auth + false + Should client authentication be used for SSL connections? + + + + submarine.server.ssl.keystore.path + keystore + Path to keystore relative to submarine configuration directory + + + + submarine.server.ssl.keystore.type + JKS + The format of the given keystore (e.g. JKS or PKCS12) + + + + submarine.server.ssl.keystore.password + change me + Keystore password. Can be obfuscated by the Jetty Password tool + + + + + + submarine.server.ssl.truststore.path + truststore + Path to truststore relative to submarine configuration directory. Defaults to the keystore path + + + + submarine.server.ssl.truststore.type + JKS + The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type + + + + + + workbench.web.war + ../submarine-workbench-web.war + Submarine workbench web war file path. + + + + jdbc.driverClassName + com.mysql.jdbc.Driver + + + jdbc.url + jdbc:mysql://127.0.0.1:3306/submarine?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false + + + jdbc.username + submarine + + + jdbc.password + password + + + + metastore.jdbc.url + jdbc:mysql://127.0.0.1:3306/metastore?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false + + + metastore.jdbc.username + metastore + + + metastore.jdbc.password + password + + + + submarine.runtime.class + org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory + RuntimeFactory for Submarine jobs + + + + submarine.server.rpc.enabled + false + Run jobs using rpc server. + + + + submarine.server.rpc.port + 8980 + Rpc server port + + + + submarine.submitter + k8s + The submitter which you want used in the server. Build-in k8s + + + diff --git a/conf/submarine-site.xml.template b/conf/submarine-site.xml.template new file mode 100755 index 0000000000000000000000000000000000000000..ccb2aa0ea398786a7af19fcf25d009d69c417648 --- /dev/null +++ b/conf/submarine-site.xml.template @@ -0,0 +1,163 @@ + + + + + + + + submarine.cluster.addr + + submarine cluster address list, e.g. ip1:port1;ip2:port2;ip3:port3 + + + + submarine.server.addr + 0.0.0.0 + Server address + + + + submarine.server.port + 8080 + Server port. + + + + submarine.server.ssl + false + Should SSL be used by the submarine server? + + + + submarine.server.ssl.port + 8443 + Server ssl port. (used when ssl property is set to true) + + + + submarine.server.ssl.client.auth + false + Should client authentication be used for SSL connections? + + + + submarine.server.ssl.keystore.path + keystore + Path to keystore relative to submarine configuration directory + + + + submarine.server.ssl.keystore.type + JKS + The format of the given keystore (e.g. JKS or PKCS12) + + + + submarine.server.ssl.keystore.password + change me + Keystore password. Can be obfuscated by the Jetty Password tool + + + + + + submarine.server.ssl.truststore.path + truststore + Path to truststore relative to submarine configuration directory. Defaults to the keystore path + + + + submarine.server.ssl.truststore.type + JKS + The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type + + + + + + workbench.web.war + ../submarine-workbench-web.war + Submarine workbench web war file path. + + + + jdbc.driverClassName + com.mysql.jdbc.Driver + + + jdbc.url + jdbc:mysql://127.0.0.1:3306/submarine?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false + + + jdbc.username + submarine + + + jdbc.password + password + + + + metastore.jdbc.url + jdbc:mysql://127.0.0.1:3306/metastore?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false + + + metastore.jdbc.username + metastore + + + metastore.jdbc.password + password + + + + submarine.runtime.class + org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory + RuntimeFactory for Submarine jobs + + + + submarine.server.rpc.enabled + false + Run jobs using rpc server. + + + + submarine.server.rpc.port + 8980 + Rpc server port + + + + submarine.submitter + k8s + The submitter which you want used in the server. Build-in k8s + + diff --git a/dev-support/cicd/Dockerfile b/dev-support/cicd/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..edf04025602354e65474de8031699ed60f10fb50 --- /dev/null +++ b/dev-support/cicd/Dockerfile @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 +RUN \ + apt-get update && \ + apt-get install -y wget vim git python-pip zlib1g-dev libssl-dev && \ + wget https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz && \ + tar xzf Python-2.7.13.tgz && cd Python-2.7.13 && ./configure --with-zlib && make install && make altinstall && \ + cd /root && wget https://files.pythonhosted.org/packages/59/38/55dd25a965990bd93f77eb765b189e72cf581ce1c2de651cb7b1dea74ed1/virtualenv-16.2.0.tar.gz && \ + tar xf virtualenv-16.2.0.tar.gz && \ + python virtualenv-16.2.0/virtualenv.py venv2.7 && \ + . venv2.7/bin/activate && \ + pip install jira && \ + deactivate + +RUN \ + git clone https://gitbox.apache.org/repos/asf/submarine.git && \ + cd submarine && \ + git remote rename origin apache && \ + git remote add apache-github https://github.com/apache/submarine.git +ADD \ + entry.sh /entry.sh +ENV \ + SUBMARINE_HOME=/submarine \ + PYTHON_VENV_PATH=/root + +RUN apt-get update && apt-get -y install bundler + +RUN \ + cd / && \ + git clone https://gitbox.apache.org/repos/asf/submarine-site.git && \ + cd /submarine-site && bundle update + +ENV \ + SUBMARINE_SITE=/submarine-site + +CMD /entry.sh diff --git a/dev-support/cicd/HowToRelease.md b/dev-support/cicd/HowToRelease.md new file mode 100644 index 0000000000000000000000000000000000000000..cd4d297493c9080181ea50d558e2bc721059112d --- /dev/null +++ b/dev-support/cicd/HowToRelease.md @@ -0,0 +1,198 @@ +# How To Release Apache Submarine +This document is for Apache Submarine Release Manager to do a new release. + +## 1. Preparation +If you have not already done so, generate your PGP key and append your [signing key](http://www.apache.org/dev/release-signing.html#keys-policy) to the [KEYS](https://dist.apache.org/repos/dist/release/submarine/KEYS) file. Once you commit your changes (ask for PMC's help if you cannot), they will automatically be propagated to the website. Also upload your key to a public key server if you haven't. +End users use the KEYS file to validate that releases were done by an Apache Committer. + +``` +# generate key and send keys to a keyserver +gpg --gen-key +gpg --list-sigs +gpg --keyserver pgp.mit.edu --send-keys +gpg --refresh-keys --keyserver pgp.mit.edu + + +# ask for PMC’s help to sign the key and add your key to the submarine KEYS (Only PMCs has permission). The PMC should do +gpg --keyserver pgp.mit.edu --recv-key +gpg --sign-key +svn co --depth immediates https://dist.apache.org/repos/dist apache-dist +cd apache-dist +svn update --set-depth immediates release/submarine +cd release/submarine +gpg --list-sigs >> KEYS +gpg --armor --export >> KEYS +svn ci -m "Add 's key" +``` + +## 2. Send Release Plan +It's better to send a release plan email informing code freeze date and release date. + +## 3. Clean up the JIRA +Bulk update JIRA to move out non-blocker issues by setting the target version to the new release. +Assuming we're releasing version X, use below advanced filter in [submarine issue page](https://issues.apache.org/jira/projects/SUBMARINE). For instance, if we're releasing `0.3.0`. +``` +project in ("Apache Submarine") AND "Target Version" = 0.3.0 AND statusCategory != Done +``` +Click "tools"-> "bulk update" to edit all issues: +1. Change the target version to X+1. Here it is `0.4.0` (If it doesn't exist, ask for the PMC's help to access [administer-versions](https://issues.apache.org/jira/plugins/servlet/project-config/SUBMARINE/administer-versions?status=no-filter) to add the new version). +2. Add a comment to inform contributors. Like this. `Bulk update due to releasing 0.3.0. Please change it back if you think this is a blocker.` + +Do a double-check to confirm that there are no issues found with the above filter. And send mail to the developer list informing that we should mark "Target version" to `0.4.0` when creating new JIRAs. + +## 4. Tagging +Once the JIRA is cleaned up, we can tag the candidate release with below steps: +``` +export version=0.3.0 +export cversion=$version-RC0 +export tag=release-$cversion + +git tag -s $tag -m "Release candidate - $version" +# Verify the tag is signed with your GPG key +git tag -v $tag +# Push the tag to apache repo +git push origin $tag +``` + +## 5. Build Artifacts +The submarine artifacts consists of GPG signed source code tarball, binary tarball and docker images. +``` +cd dev-support/cicd/ +./create_release.sh $version $tag +# Move the artifacts to a folder instead of /tmp/ +mv /tmp/submarine-release ~/ +``` +> Note: In here we use the `version` not the `cversion`. + +## 6. Upload Artifacts For Vote +Before the uploading, we need to do some basic testing for the release candidates. For instance, build from the source tarball and run feature test using the binary tarball. + +### 6.1 Staging Source and Binary Tarball to self FTP server +``` +cd ~/submarine-release +sftp home.apache.org +# if not exits, please mkdir +cd public_html +mkdir submarine-$cversion && cd $_ +put -r . +exit +``` + +### 6.2 Staging Docker Images +When doing release, the release manager might needs to package an artifact candidates in this docker image and public the image candidate for a vote. +In this scenario, we can do this: + +Put submarine candidate artifacts to a folder like "~/releases/submarine-release" +``` +$ ls $release_candidates_path +submarine-dist-0.3.0-hadoop-2.9.tar.gz submarine-dist-0.3.0-src.tar.gz.asc +submarine-dist-0.3.0-hadoop-2.9.tar.gz.asc submarine-dist-0.3.0-src.tar.gz.sha512 +submarine-dist-0.3.0-hadoop-2.9.tar.gz.sha512 submarine-dist-0.3.0-src.tar.gz +``` +``` +export submarine_version=0.3.0 +export release_candidates_path=~/releases/submarine-release +./build_mini-submarine.sh +#docker run -it -h submarine-dev --net=bridge --privileged -P local/mini-submarine:0.3.0 /bin/bash +docker tag local/mini-submarine:0.3.0 apache/submarine:mini-0.3.0-RC0 +``` +In the container, we can verify that the submarine jar version is the expected 0.3.0. Then we can upload this image with a "RC" tag for a vote. + +Note: if you don't have permission to push image to docker hub, create a jira ticket to request the push permission. + +Refer to https://issues.apache.org/jira/browse/INFRA-20364 +``` +docker push apache/submarine:mini-0.3.0-RC0 +``` + +TODO: build the other images by manual. + +### 6.3 Publish Jars To Apache Maven Staging Repository +``` +export GPG_PASSPHRASE=yourPassphase +export ASF_USERID=yourApacheId +export ASF_PASSWORD=yourApachePwd +./publish_release.sh $version $tag +``` +Then to view the staging repo, we can login the https://repository.apache.org with the apache id. Click the "Staging Repositories" in the left side of the web page. And click "orgapachesubmarine-1001", then you will see the details of the repo including the URI. The URI is like this: +"https://repository.apache.org/content/repositories/orgapachesubmarine-1001" + +### 6.4 Call A Vote For The Release Candidate +After the artifacts and images are staged, we can send a vote email to the community. It's recommended that we paste the URI of RC tag, RC release artifacts, docker images, maven staging repository and the KEYS. +Refer to [here](https://www.mail-archive.com/dev@submarine.apache.org/msg01498.html) for an example. + +## 7. Release +In several days if the [vote passes](http://hadoop.apache.org/bylaws#Decision+Making), we can publish the release. If the vote fails, then we need to start another RC from [cleanup](#3.-Clean-up-the-JIRA) to the [staging](#6.-Upload-Artifacts-For-Vote). + +1. Access [submarine project version page](https://issues.apache.org/jira/projects/SUBMARINE?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page&status=unreleased). Click the version to be released, and then click the "Release" button. It will request the release date. We can fill it with the end-of-vote date. + +2. Tag the release +``` +# please replace the version to the right version +export version=0.3.0 +export tag=rel/release-$version +git tag -s $tag -m "Submarine ${version} release" +git push origin $tag +``` + +3. Copy release artifacts to apache dist server +``` +svn co --depth immediates https://dist.apache.org/repos/dist apache-dist +cd apache-dist +svn update --set-depth infinity dev/submarine +svn update --set-depth infinity release/submarine +cd dev/submarine +mkdir submarine-$version +cp ~/submarine-release/* submarine-$version/ +svn add submarine-$version +svn ci -m "Publishing the bits for submarine release ${version} to apache dist dev folder" +cd ../../ +svn mv dev/submarine/submarine-$version release/submarine/ +svn ci -m "Publishing the bits for submarine release ${version}" +``` + +Usually binary tarball becomes larger than 300MB, so it cannot be directly uploaded to the distribution directory. We can use the dev directory (https://dist.apache.org/repos/dist/dev/submarine/) first and then move it to the distribution directory by svn move + +4. In [Nexus](https://repository.apache.org/), effect the release of artifacts by selecting the staged repository and then clicking Release. If there were multiple RCs, simply drop the staging repositories corresponding to failed RCs. + +5. Upload the docker images +``` +docker tag apache/submarine:mini-0.3.0-RC0 apache/submarine:mini-0.3.0 +docker push apache/submarine:mini-0.3.0 +``` + +6. Update the version in pom.xml +``` +# if the new version a point release +mvn versions:set -DgenerateBackupPoms=false -DnewVersion=X.(Y+1).Z-SNAPSHOT +git commit -a -m "Preparing for X.(Y+1).Z development" +``` + +7. Wait 24 hours for release to propagate to mirrors. + +8. Update the website. The guide is [here](https://github.com/apache/submarine-site) +``` +git clone https://github.com/apache/submarine-site.git +cd submarine-site +git checkout master +# Edit download.md to add the new release content +# The url of the binary and source tarball should be like +# "https://www.apache.org/dyn/closer.cgi/submarine/submarine-0.3.0/submarine-dist-0.3.0-hadoop-2.9.tar.gz" +# due to it will find a mirror server for different region. +# The url of the signature and checksum should be like +# "https://www.apache.org/dist/submarine/submarine-0.3.0/submarine-dist-0.3.0-hadoop-2.9.tar.gz.sha512)" +# due to they're not mirrored from apache dist server. +# For the release note web page, we need to create a MD file under "releases" directory +# like "submarine-release-0.3.0.md". The count of issues can be found +# from JIRA like "https://issues.apache.org/jira/projects/SUBMARINE/versions/12345556". +# And the details of issues can also get from "Release Notes" in that page. +cp release/submarine-release-0.3.0.md release/submarine-release-.md +vim release/submarine-release-.md +cd .. +docker run -it -p 4000:4000 -v $PWD/submarine-site:/submarine-site hadoopsubmarine/submarine-website:1.0.0 bash +cd /submarine-site +bundle exec jekyll serve --watch --host=0.0.0.0 +# Open another terminal, you can edit MD files and refresh the webpage to see changes instantly. +``` + +9. Send announcements to the user and developer lists once the site changes are visible. diff --git a/dev-support/cicd/README.md b/dev-support/cicd/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4ed4c3a49454ac35a3f47246693edb3ec492ef13 --- /dev/null +++ b/dev-support/cicd/README.md @@ -0,0 +1,80 @@ + +# CICD Introduction + +> Please note that cicd is a tool provided to submarine committers for PR merging and release. Only submarine committers have permission to execute. + +This cicd directory contains several scripts to ease code and release management. For how-to-release, check [here](./HowToRelease.md) + +To use them more easily, we provide a Docker image to help committer to handle tasks like committing code and release build. + +## Docker mode + +``` +./build_and_start_cicd_image.sh +``` + +Or + +``` +cd /dev-support/cicd +docker build -t submarine-cicd . +docker run -it --rm submarine-cicd +``` + +Jira username, password, apache id and apache username are required in the docker container. You can +add them to the local environment variable. + +``` +vi ~/.bash_profile +export JIRA_USERNAME='Your jira username' +export JIRA_PASSWORD='Your jira password' +export APACHE_ID='Your apache id' # Your apache email prefix +export APACHE_NAME='Your apache name' +``` + +And you'll see output like below and then you can decide what to accomplish. +``` +$ docker run -it -e JIRA_USERNAME="${JIRA_USERNAME}" -e JIRA_PASSWORD="${JIRA_PASSWORD}" -e APACHE_ID="${APACHE_ID}" -e APACHE_NAME="${APACHE_NAME}" -p 4000:4000 --rm submarine-cicd +``` + +The screen outputs the following information: + +``` +Menu: + 1. Merge PR + 2. Update Submarine Website +Enter Menu ID: +``` + +As you can see, the Docker mode support several features like merging PR and updating the web site. Choose the task you need to do and follow the popup tip to go on. + +## Manual mode (Not Recommended) + +First, You need install `python 2.7.13` and `pip insall jira` + +### The Procedure of Merging PR + +1. mkdir ${work_dir} +2. cd ${work_dir} +3. git clone https://gitbox.apache.org/repos/asf/submarine.git +4. cd submarine +5. git remote rename origin apache +6. git remote add apache-github https://github.com/apache/submarine.git +7. optional: git config --local --add user.name {name} +8. optional: git config --local --add user.email {username}@apache.org +9. optional: echo -e "JIRA_USERNAME={jira_username}\nJIRA_PASSWORD={jira_password}" >> ~/.bashrc +10. optional: source ~/.bashrc +11. dev-support/cicd/merge_submarine_pr.py + diff --git a/dev-support/cicd/build_and_start_cicd_image.sh b/dev-support/cicd/build_and_start_cicd_image.sh new file mode 100755 index 0000000000000000000000000000000000000000..a43600faf6650cbadd97ea16442330a884842aa0 --- /dev/null +++ b/dev-support/cicd/build_and_start_cicd_image.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -euo pipefail + +printf "Building Submarine CI/CD Image.\n" +docker build -t submarine-cicd . +printf "Start Submarine CI/CD.\n" +docker run -it --rm -p 4000:4000 submarine-cicd diff --git a/dev-support/cicd/common_release.sh b/dev-support/cicd/common_release.sh new file mode 100755 index 0000000000000000000000000000000000000000..517bc372ea938b7095ee6b76f50e88cbeda35260 --- /dev/null +++ b/dev-support/cicd/common_release.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# common functions + +if [[ -z "${TAR:-}" ]]; then + TAR="/usr/bin/tar" +fi + +if [[ -z "${SHASUM:-}" ]]; then + SHASUM="/usr/bin/shasum" +fi + +if [[ -z "${WORKING_DIR:-}" ]]; then + WORKING_DIR="/tmp/submarine-release" +fi + +DEBUG_SUBMARINE_SCRIPT=false +if $DEBUG_SUBMARINE_SCRIPT; then + echo "DEBUGGING, skip remove ${WORKING_DIR}" +else + echo "Cleaning up ${WORKING_DIR}" + rm -rf "${WORKING_DIR}" + mkdir "${WORKING_DIR}" +fi + +# If set to 'yes', release script will deploy artifacts to SNAPSHOT repository. +DO_SNAPSHOT='no' + +usage() { + echo "usage) $0 [Release version] [Branch or Tag]" + echo " ex. $0 0.6.0 v0.6.0" + exit 1 +} + +function git_clone() { + echo "Clone the source" + # clone source + git clone https://git-wip-us.apache.org/repos/asf/submarine.git "${WORKING_DIR}/submarine" + + if [[ $? -ne 0 ]]; then + echo "Can not clone source repository" + exit 1 + fi + + cd "${WORKING_DIR}/submarine" + git checkout "${GIT_TAG}" + echo "Checked out ${GIT_TAG}" +} diff --git a/dev-support/cicd/create_release.sh b/dev-support/cicd/create_release.sh new file mode 100755 index 0000000000000000000000000000000000000000..fd83ef5ccb7b591ddbf9677183f55d8d0b5b10a0 --- /dev/null +++ b/dev-support/cicd/create_release.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The script helps making a release. +# You need to specify release version and branch|tag name. +# +# Here are some helpful documents for the release. +# http://www.apache.org/dev/release.html +# http://www.apache.org/dev/release-publishing +# http://www.apache.org/dev/release-signing.html + +set -euo pipefail + +BASEDIR="$(dirname "$0")" +. "${BASEDIR}/common_release.sh" +echo "${BASEDIR}/common_release.sh" + +if [[ $# -ne 2 ]]; then + usage +fi + +for var in GPG_PASSPHRASE; do + if [[ -z "${!var:-}" ]]; then + echo "You need ${var} variable set" + exit 1 + fi +done + +RELEASE_VERSION="$1" +GIT_TAG="$2" + +function compile_src_and_bin() { + cd ${WORKING_DIR}/submarine + echo "mvn versions:set -DnewVersion=${RELEASE_VERSION}" + mvn versions:set -DnewVersion="${RELEASE_VERSION}" + echo "mvn clean package -DskipTests -Psrc" + mvn clean package -DskipTests -Psrc + if [[ $? -ne 0 ]]; then + echo "Build failed. ${BUILD_FLAGS:-}" + exit 1 + fi +} + +function make_source_package() { + # create source package + cd ${WORKING_DIR}/submarine/submarine-dist/target + cd submarine-dist-*-src + # remove unneeded dir .github + rm -rf submarine-dist-${RELEASE_VERSION}-src/.github + ${TAR} cfz "submarine-dist-${RELEASE_VERSION}-src.tar.gz" "submarine-dist-${RELEASE_VERSION}-src" + mv "submarine-dist-${RELEASE_VERSION}-src.tar.gz" ${WORKING_DIR} + echo "Signing the source package" + cd "${WORKING_DIR}" + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --armor \ + --output "submarine-dist-${RELEASE_VERSION}-src.tar.gz.asc" \ + --detach-sig "${WORKING_DIR}/submarine-dist-${RELEASE_VERSION}-src.tar.gz" + ${SHASUM} -a 512 "submarine-dist-${RELEASE_VERSION}-src.tar.gz" > \ + "${WORKING_DIR}/submarine-dist-${RELEASE_VERSION}-src.tar.gz.sha512" +} + +function make_binary_release() { + R_DIR_NAME=submarine-dist-${RELEASE_VERSION}-hadoop-2.9 + cd ${WORKING_DIR}/submarine/submarine-dist/target + mv "${R_DIR_NAME}.tar.gz" ${WORKING_DIR} + # sign bin package + cd ${WORKING_DIR} + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --armor \ + --output "${R_DIR_NAME}.tar.gz.asc" \ + --detach-sig "${R_DIR_NAME}.tar.gz" + ${SHASUM} -a 512 "${R_DIR_NAME}.tar.gz" > \ + "${R_DIR_NAME}.tar.gz.sha512" +} + +if [ -d "${WORKING_DIR}/submarine/submarine-dist/target" ]; then + if $DEBUG_SUBMARINE_SCRIPT; then + echo "DEBUGGING, skip re-building submarine" + fi +else + git_clone + compile_src_and_bin +fi + +make_source_package +make_binary_release + +# remove non release files and dirs +echo "Deleting ${WORKING_DIR}/submarine" +rm -rf "${WORKING_DIR}/submarine" +echo "Release files are created under ${WORKING_DIR}" diff --git a/dev-support/cicd/entry.sh b/dev-support/cicd/entry.sh new file mode 100755 index 0000000000000000000000000000000000000000..811683fc9b7e22ae3aac382442e7c64d56152e1a --- /dev/null +++ b/dev-support/cicd/entry.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -euo pipefail +# activate python 2.7.13 environment +. ${PYTHON_VENV_PATH}/venv2.7/bin/activate + +function start_menu(){ + printf "Menu:\n" + printf "\t1. Merge PR\n" + printf "\t2. Update Submarine Website\n" + read -p "Enter Menu ID:" menu_id + case $menu_id in + "1") + merge_pr + ;; + "2") + update_submarine_site + ;; + "*") + printf "unknown. Exiting." + ;; + esac +} + +function merge_pr(){ + printf "==== Merge PR Begin ====\n" + jira_name="n" + jira_pwd="p" + apache_id="id" + apache_name="name" + + if [ -z "${JIRA_USERNAME:-}" ]; then + read -p "Enter Your Apache JIRA User name: " jira_name + else + jira_name=$JIRA_USERNAME + fi + echo "Got JIRA name: ${jira_name}" + + if [ -z "${JIRA_PASSWORD:-}" ]; then + read -s -p "Enter Your Apache JIRA User passwd: " jira_pwd + else + jira_pwd=$JIRA_PASSWORD + fi + + if [ -z "${APACHE_ID:-}" ]; then + printf "\n" + read -p "Enter Your Apache committer ID: " apache_id + else + apache_id=$APACHE_ID + fi + echo "Got Apache ID: ${apache_id}" + + if [ -z "${APACHE_NAME:-}" ]; then + read -p "Enter Your Apache committer name: " apache_name + else + apache_name=$APACHE_NAME + fi + echo "Got Apache name: ${apache_name}" + + cd $SUBMARINE_HOME + git checkout master + git pull + git config user.name "${apache_name}" + git config user.email "${apache_id}@apache.org" + export JIRA_USERNAME=${jira_name} + export JIRA_PASSWORD=${jira_pwd} + python dev-support/cicd/merge_submarine_pr.py + printf "==== Merge PR END ====\n" +} + +function update_submarine_site(){ + printf "==== Update Submarine Site Begin ====\n" + apache_id="id" + apache_name="name" + + if [ -z "${APACHE_ID:-}" ]; then + printf "\n" + read -p "Enter Your Apache committer ID: " apache_id + else + apache_id=$APACHE_ID + fi + echo "Got Apache ID: ${apache_id}" + + if [ -z "${APACHE_NAME:-}" ]; then + read -p "Enter Your Apache committer name: " apache_name + else + apache_name=$APACHE_NAME + fi + echo "Got Apache name: ${apache_name}" + + cd $SUBMARINE_SITE + git checkout master + git pull + git config user.name "${apache_name}" + git config user.email "${apache_id}@apache.org" + git config credential.helper store + bundle update + bundle exec jekyll serve --watch --host=0.0.0.0 > /tmp/jekyll.log 2>&1 & + echo "==== Please use vim to edit md files and check http://localhost:4000/ for the update ====" + while true; do + echo "==== Edit Mode: Type 'exit' when you finish the changes (you don't need to perform git commit/push). ====" + bash + read -p "Have you finished updating the MD files? y/n/quit " commit + case $commit in + "y") + echo "Start committing changes.." + cd $SUBMARINE_SITE + git add . + git status + read -p "Please input the commit message: " message + git commit -m "${message} (master branch)" + git push origin master + cp -r _site /_site + git checkout asf-site + cp -r /_site/* ./ + git add . + git status + git commit -m "${message} (asf-site branch)" + git push origin asf-site + echo "Exiting edit mode.." + break + ;; + "n") + continue + ;; + "quit") + printf "Exiting edit mode.." + break + ;; + "q") + printf "Exiting edit mode.." + break + ;; + "*") + printf "Unknown. Exiting edit mode.." + break + ;; + esac + done + printf "\n" + printf "==== Update Submarine Site END ====\n" + echo "==== Enter shell again incase any unexpected error happens ====" + bash + echo "Exiting CICD.." +} + +start_menu +deactivate diff --git a/dev-support/cicd/merge_submarine_pr.py b/dev-support/cicd/merge_submarine_pr.py new file mode 100755 index 0000000000000000000000000000000000000000..7b5e57c616793f451fb449e8a759b7260e9de702 --- /dev/null +++ b/dev-support/cicd/merge_submarine_pr.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Utility for creating well-formed pull request merges and pushing them to Apache. +# usage: ./merge-submarine-pr.py (see config env vars below) +# +# This utility assumes you already have local a Submarine git folder and that you +# have added remotes corresponding to both (i) the github apache Submarine +# mirror and (ii) the apache git repo. + +import json +import os +import re +import subprocess +import sys +import urllib2 + +try: + import jira.client + JIRA_IMPORTED = True +except ImportError: + JIRA_IMPORTED = False + +# Location of your Submarine git development area +SUBMARINE_HOME = os.environ.get("SUBMARINE_HOME", os.getcwd()) +# Remote name which points to the Github site +PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github") +# Remote name which points to Apache git +PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache") +# ASF JIRA username +JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "you-jira-user-name") +# ASF JIRA password +JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", "you-jira-password") + +GITHUB_BASE = "https://github.com/apache/submarine/pull" +GITHUB_API_BASE = "https://api.github.com/repos/apache/submarine" +JIRA_BASE = "https://issues.apache.org/jira/browse" +JIRA_API_BASE = "https://issues.apache.org/jira" +# Prefix added to temporary branches +BRANCH_PREFIX = "PR_TOOL" + +os.chdir(SUBMARINE_HOME) + + +def get_json(url): + try: + return json.load(urllib2.urlopen(url)) + except urllib2.HTTPError as e: + print "Unable to fetch URL, exiting: %s" % url + sys.exit(-1) + + +def fail(msg): + print msg + clean_up() + sys.exit(-1) + + +def run_cmd(cmd): + print cmd + if isinstance(cmd, list): + return subprocess.check_output(cmd) + else: + return subprocess.check_output(cmd.split(" ")) + + +def continue_maybe(prompt): + result = raw_input("\n%s (y/n): " % prompt) + if result.lower() != "y": + fail("Okay, exiting") + + +original_head = run_cmd("git rev-parse HEAD")[:8] + + +def clean_up(): + print "Restoring head pointer to %s" % original_head + run_cmd("git checkout %s" % original_head) + + branches = run_cmd("git branch").replace(" ", "").split("\n") + + for branch in filter(lambda x: x.startswith(BRANCH_PREFIX), branches): + print "Deleting local branch %s" % branch + run_cmd("git branch -D %s" % branch) + + +# merge the requested PR and return the merge hash +def merge_pr(pr_num, target_ref): + pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num) + target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper()) + run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name)) + run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name)) + run_cmd("git checkout %s" % target_branch_name) + + had_conflicts = False + try: + run_cmd(['git', 'merge', pr_branch_name, '--squash']) + except Exception as e: + msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e + continue_maybe(msg) + msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?" + continue_maybe(msg) + had_conflicts = True + + commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, + '--pretty=format:%an <%ae>']).split("\n") + commit_date = run_cmd(['git', 'log', '%s' % pr_branch_name, '-1', + '--pretty=format:%ad']) + distinct_authors = sorted(set(commit_authors), + key=lambda x: commit_authors.count(x), reverse=True) + primary_author = distinct_authors[0] + commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, + '--pretty=format:%h [%an] %s']).split("\n\n") + + merge_message_flags = [] + + merge_message_flags += ["-m", title] + if body is not None: + # We remove @ symbols from the body to avoid triggering e-mails + # to people every time someone creates a public fork of Submarine. + if isinstance(body, unicode): + merge_message_flags += ["-m", body.encode("utf-8").replace("@", "")] + else: + merge_message_flags += ["-m", body.replace("@", "")] + + authors = "\n".join(["Author: %s" % a for a in distinct_authors]) + + merge_message_flags += ["-m", authors] + + if had_conflicts: + committer_name = run_cmd("git config --get user.name").strip() + committer_email = run_cmd("git config --get user.email").strip() + message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % ( + committer_name, committer_email) + merge_message_flags += ["-m", message] + + # The string "Closes #%s" string is required for GitHub to correctly close the PR + merge_message_flags += [ + "-m", + "Closes #%s from %s and squashes the following commits:" % (pr_num, pr_repo_desc)] + for c in commits: + merge_message_flags += ["-m", c] + + run_cmd(['git', 'commit', '--author="%s"' % primary_author, '--date="%s"' % commit_date] + merge_message_flags) + + continue_maybe("Merge complete (local ref %s). Push to %s?" % ( + target_branch_name, PUSH_REMOTE_NAME)) + + try: + run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, target_ref)) + except Exception as e: + clean_up() + fail("Exception while pushing: %s" % e) + + merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8] + clean_up() + print("Pull request #%s merged!" % pr_num) + print("Merge hash: %s" % merge_hash) + return merge_hash + + +def cherry_pick(pr_num, merge_hash, default_branch): + pick_ref = raw_input("Enter a branch name [%s]: " % default_branch) + if pick_ref == "": + pick_ref = default_branch + + pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper()) + + run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name)) + run_cmd("git checkout %s" % pick_branch_name) + + try: + run_cmd("git cherry-pick -sx %s" % merge_hash) + except Exception as e: + msg = "Error cherry-picking: %s\nWould you like to manually fix-up this merge?" % e + continue_maybe(msg) + msg = "Okay, please fix any conflicts and finish the cherry-pick. Finished?" + continue_maybe(msg) + + continue_maybe("Pick complete (local ref %s). Push to %s?" % ( + pick_branch_name, PUSH_REMOTE_NAME)) + + try: + run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, pick_ref)) + except Exception as e: + clean_up() + fail("Exception while pushing: %s" % e) + + pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8] + clean_up() + + print("Pull request #%s picked into %s!" % (pr_num, pick_ref)) + print("Pick hash: %s" % pick_hash) + return pick_ref + + +def fix_version_from_branch(branch, versions): + # Note: Assumes this is a sorted (newest->oldest) list of un-released versions + if branch == "master": + return versions[0] + else: + branch_ver = branch.replace("branch-", "") + return filter(lambda x: x.name.startswith(branch_ver), versions)[-1] + + +def resolve_jira_issue(merge_branches, comment, default_jira_id=""): + asf_jira = jira.client.JIRA({'server': JIRA_API_BASE}, + basic_auth=(JIRA_USERNAME, JIRA_PASSWORD)) + + jira_id = raw_input("Enter a JIRA id [%s]: " % default_jira_id) + if jira_id == "": + jira_id = default_jira_id + + try: + issue = asf_jira.issue(jira_id) + except Exception as e: + fail("ASF JIRA could not find %s\n%s" % (jira_id, e)) + + cur_status = issue.fields.status.name + cur_summary = issue.fields.summary + cur_assignee = issue.fields.assignee + if cur_assignee is None: + cur_assignee = "NOT ASSIGNED!!!" + else: + cur_assignee = cur_assignee.displayName + + if cur_status == "Resolved" or cur_status == "Closed": + fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status)) + print ("=== JIRA %s ===" % jira_id) + print ("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % ( + cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id)) + + versions = asf_jira.project_versions("SUBMARINE") + versions = sorted(versions, key=lambda x: x.name, reverse=True) + versions = filter(lambda x: x.raw['released'] is False, versions) + # Consider only x.y.z versions + versions = filter(lambda x: re.match('\d+\.\d+\.\d+', x.name), versions) + + default_fix_versions = map(lambda x: fix_version_from_branch(x, versions).name, merge_branches) + for v in default_fix_versions: + # Handles the case where we have forked a release branch but not yet made the release. + # In this case, if the PR is committed to the master branch and the release branch, we + # only consider the release branch to be the fix version. E.g. it is not valid to have + # both 1.1.0 and 1.0.0 as fix versions. + (major, minor, patch) = v.split(".") + if patch == "0": + previous = "%s.%s.%s" % (major, int(minor) - 1, 0) + if previous in default_fix_versions: + default_fix_versions = filter(lambda x: x != v, default_fix_versions) + default_fix_versions = ",".join(default_fix_versions) + + fix_versions = raw_input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions) + if fix_versions == "": + fix_versions = default_fix_versions + fix_versions = fix_versions.replace(" ", "").split(",") + + def get_version_json(version_str): + return filter(lambda v: v.name == version_str, versions)[0].raw + + jira_fix_versions = map(lambda v: get_version_json(v), fix_versions) + + resolve = filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id))[0] + asf_jira.transition_issue( + jira_id, resolve["id"], fixVersions=jira_fix_versions, comment=comment) + + print "Successfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions) + + +def resolve_jira_issues(title, merge_branches, comment): + jira_ids = re.findall("SUBMARINE-[0-9]{3,5}", title) + + if len(jira_ids) == 0: + resolve_jira_issue(merge_branches, comment) + for jira_id in jira_ids: + resolve_jira_issue(merge_branches, comment, jira_id) + + +#branches = get_json("%s/branches" % GITHUB_API_BASE) +#branch_names = filter(lambda x: x.startswith("branch-"), [x['name'] for x in branches]) +# Assumes branch names can be sorted lexicographically +#latest_branch = sorted(branch_names, reverse=True)[0] +latest_branch = "master" + +pr_num = raw_input("Which pull request would you like to merge? (e.g. 23): ") +pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num)) +pr_events = get_json("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num)) + +url = pr["url"] +title = pr["title"] +body = pr["body"] +target_ref = pr["base"]["ref"] +user_login = pr["user"]["login"] +base_ref = pr["head"]["ref"] +pr_repo_desc = "%s/%s" % (user_login, base_ref) + +# Merged pull requests don't appear as merged in the GitHub API; +# Instead, they're closed by asfgit. +merge_commits = \ + [e for e in pr_events if e["actor"]["login"] == "asfgit" and e["event"] == "closed"] + +if merge_commits: + merge_hash = merge_commits[0]["commit_id"] + message = get_json("%s/commits/%s" % (GITHUB_API_BASE, merge_hash))["commit"]["message"] + + print "Pull request %s has already been merged, assuming you want to backport" % pr_num + commit_is_downloaded = run_cmd(['git', 'rev-parse', '--quiet', '--verify', + "%s^{commit}" % merge_hash]).strip() != "" + if not commit_is_downloaded: + fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num) + + print "Found commit %s:\n%s" % (merge_hash, message) + cherry_pick(pr_num, merge_hash, latest_branch) + sys.exit(0) + +if not bool(pr["mergeable"]): + msg = "Pull request %s is not mergeable in its current form.\n" % pr_num + \ + "Continue? (experts only!)" + continue_maybe(msg) + +print ("\n=== Pull Request #%s ===" % pr_num) +print ("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" % ( + title, pr_repo_desc, target_ref, url)) +continue_maybe("Proceed with merging pull request #%s?" % pr_num) + +merged_refs = [target_ref] + +merge_hash = merge_pr(pr_num, target_ref) + +pick_prompt = "Would you like to pick %s into another branch?" % merge_hash +while raw_input("\n%s (y/n): " % pick_prompt).lower() == "y": + merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)] + +if JIRA_IMPORTED: + if JIRA_USERNAME and JIRA_PASSWORD: + continue_maybe("Would you like to update an associated JIRA?") + jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (pr_num, GITHUB_BASE, pr_num) + resolve_jira_issues(title, merged_refs, jira_comment) + else: + print "JIRA_USERNAME and JIRA_PASSWORD not set" + print "Exiting without trying to close the associated JIRA." +else: + print "Could not find jira library. Run 'sudo pip install jira' to install." + print "Exiting without trying to close the associated JIRA." diff --git a/dev-support/cicd/publish_release.sh b/dev-support/cicd/publish_release.sh new file mode 100755 index 0000000000000000000000000000000000000000..ac620f14325962097984f2b963649ea53e62e67a --- /dev/null +++ b/dev-support/cicd/publish_release.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The script helps publishing release to maven. +# You need to specify release version and branch|tag name. +# +# Here's some helpful documents for the release. +# http://www.apache.org/dev/publishing-maven-artifacts.html +set -euo pipefail +BASEDIR="$(dirname "$0")" +. "${BASEDIR}/common_release.sh" + +if [[ $# -ne 2 ]]; then + usage +fi + +for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD; do + if [[ -z "${!var:-}" ]]; then + echo "You need ${var} variable set" + exit 1 + fi +done + +export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512m" +RED='\033[0;31m' +NC='\033[0m' # No Color + +RELEASE_VERSION="$1" +GIT_TAG="$2" +if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then + DO_SNAPSHOT="yes" +fi + +NEXUS_STAGING="https://repository.apache.org/service/local/staging" +# Get this from apache infra admin +NEXUS_PROFILE="2525cde13fad2a" + +function cleanup() { + echo "Remove working directory and maven local repository" + rm -rf ${WORKING_DIR} +} + +function curl_error() { + ret=${1} + if [[ $ret -ne 0 ]]; then + echo "curl response code is: ($ret)" + echo "See https://curl.haxx.se/libcurl/c/libcurl-errors.html to know the detailed cause of error." + echo -e "${RED}Failed to publish maven artifact to staging repository." + echo -e "IMPORTANT: You will have to re-run publish_release.sh to complete maven artifact publish.${NC}" + cleanup + exit 1 + fi +} + + +# +# Publishing Apache Submarine artifact to Apache snapshot repository. +# +function publish_snapshot_to_maven() { + cd "${WORKING_DIR}/submarine" + echo "Deploying Apache Submarine $RELEASE_VERSION version to snapshot repository." + + if [[ ! $RELEASE_VERSION == *"SNAPSHOT"* ]]; then + echo "ERROR: Snapshots must have a version containing 'SNAPSHOT'" + echo "ERROR: You gave version '$RELEASE_VERSION'" + exit 1 + fi + + tmp_repo="$(mktemp -d /tmp/submarine-repo-XXXXX)" + mvn versions:set -DnewVersion=$RELEASE_VERSION + tmp_settings="tmp-settings.xml" + echo "" > $tmp_settings + echo "apache.snapshots.https$ASF_USERID" >> $tmp_settings + echo "$ASF_PASSWORD" >> $tmp_settings + echo "" >> $tmp_settings + + mvn --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -DskipTests deploy + + rm $tmp_settings + rm -rf $tmp_repo +} + +function publish_to_maven() { + cd "${WORKING_DIR}/submarine" + # Force release version + mvn versions:set -DnewVersion="${RELEASE_VERSION}" + + # Using Nexus API documented here: + # https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Staging-Repository-via-REST-API + echo "Creating Nexus staging repository" + repo_request="Apache Submarine ${RELEASE_VERSION}" + out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \ + -H 'Content-Type:application/xml' -v \ + "${NEXUS_STAGING}/profiles/${NEXUS_PROFILE}/start")" + create_ret=$? + curl_error $create_ret + staged_repo_id="$(echo ${out} | sed -e 's/.*\(orgapachesubmarine-[0-9]\{4\}\).*/\1/')" + if [[ -z "${staged_repo_id}" ]]; then + echo "Fail to create staging repository" + exit 1 + fi + + echo "Created Nexus staging repository: ${staged_repo_id}" + + rm -rf $HOME/.m2/repository/org/apache/submarine + + # build and install to local + echo "mvn clean install -DskipTests" + mvn clean install -DskipTests + if [[ $? -ne 0 ]]; then + echo "Mvn install failed." + exit 1 + fi + + pushd "${HOME}/.m2/repository/org/apache/submarine" + find . -type f | grep -v '\.jar$' | grep -v '\.pom$' |grep -v '\.war$' | xargs rm + + echo "Creating hash and signature files" + for file in $(find . -type f); do + echo "${GPG_PASSPHRASE}" | gpg --passphrase-fd 0 --output "${file}.asc" \ + --detach-sig --armor "${file}" + if [ $(command -v md5) ]; then + # Available on OS X; -q to keep only hash + md5 -q $file > $file.md5 + else + # Available on Linux; cut to keep only hash + md5sum $file | cut -f1 -d' ' > $file.md5 + fi + ${SHASUM} -a 1 "${file}" | cut -f1 -d' ' > "${file}.sha1" + done + + nexus_upload="${NEXUS_STAGING}/deployByRepositoryId/${staged_repo_id}" + echo "Uploading files to ${nexus_upload}" + for file in $(find . -type f); do + # strip leading ./ + file_short="$(echo "${file}" | sed -e 's/\.\///')" + dest_url="${nexus_upload}/org/apache/submarine/$file_short" + echo " Uploading ${file_short}" + curl -u "${ASF_USERID}:${ASF_PASSWORD}" --upload-file "${file_short}" "${dest_url}" + upload_ret=$? + curl_error $upload_ret + done + + echo "Closing nexus staging repository" + repo_request="${staged_repo_id}Apache Submarine ${RELEASE_VERSION}" + out="$(curl -X POST -d "${repo_request}" -u "${ASF_USERID}:${ASF_PASSWORD}" \ + -H 'Content-Type:application/xml' -v \ + "${NEXUS_STAGING}/profiles/${NEXUS_PROFILE}/finish")" + close_ret=$? + curl_error $close_ret + echo "Closed Nexus staging repository: ${staged_repo_id}" + popd + echo "Complete publishing maven artifacts to apache staging repository" + echo "Once release candidate pass the vote, do not forget to hit the release button in https://repository.apache.org" +} + +git_clone +if [[ "${DO_SNAPSHOT:-}" == 'yes' ]]; then + publish_snapshot_to_maven +else + publish_to_maven +fi +cleanup diff --git a/dev-support/docker-images/database/Dockerfile b/dev-support/docker-images/database/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..996bd23c61398f6fb11ac8a6ba35a33c97b16e96 --- /dev/null +++ b/dev-support/docker-images/database/Dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM submarinehub/mysql:5.7.27 +MAINTAINER Apache Software Foundation + +ENV AUTO_RUN_DIR /docker-entrypoint-initdb.d + +ADD database/* /tmp/database/ +ADD startup.sh ${AUTO_RUN_DIR}/ +RUN chmod +x ${AUTO_RUN_DIR}/startup.sh + +# mysql port +EXPOSE 3306 diff --git a/dev-support/docker-images/database/build.sh b/dev-support/docker-images/database/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..d87c6684129cb3f0a24b857f254754a69a4f5cee --- /dev/null +++ b/dev-support/docker-images/database/build.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +if [ -L ${BASH_SOURCE-$0} ]; then + PWD=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + PWD=$(dirname ${BASH_SOURCE-$0}) +fi +export CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd) +SUBMARINE_HOME=${CURRENT_PATH}/../../.. + +SUBMARINE_VERSION="0.5.0" +SUBMARINE_IMAGE_NAME="apache/submarine:database-${SUBMARINE_VERSION}" + +cp -rf "${SUBMARINE_HOME}/docs/database" "${CURRENT_PATH}" + +# build image +echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." +cd ${CURRENT_PATH} +docker build -t ${SUBMARINE_IMAGE_NAME} . + +# clean template file +rm -rf ${CURRENT_PATH}/database diff --git a/dev-support/docker-images/database/startup.sh b/dev-support/docker-images/database/startup.sh new file mode 100755 index 0000000000000000000000000000000000000000..884fab6008def948a548f3e47bf16308d057688c --- /dev/null +++ b/dev-support/docker-images/database/startup.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql -uroot -p$MYSQL_ROOT_PASSWORD <> /etc/environment && \ + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ + echo "LANG=en_US.UTF-8" > /etc/locale.conf && \ + locale-gen en_US.UTF-8 + +ENV NB_USER=$NB_USER \ + NB_UID=$NB_UID \ + NB_PREFIX=$NB_PREFIX \ + NB_PORT=$NB_PORT \ + CONDA_DIR=/opt/conda \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH=$CONDA_DIR/bin:$PATH \ + HOME=/home/$NB_USER + +# Create NB_USER user with UID=1000 and in the 'users' group +RUN useradd -M -s /bin/bash -N -u $NB_UID $NB_USER && \ + chown -R ${NB_USER}:users /usr/local/bin && \ + mkdir -p $HOME && \ + chown -R ${NB_USER}:users ${HOME} && \ + mkdir -p ${CONDA_DIR} && \ + chown -R ${NB_USER}:users ${CONDA_DIR} && \ + chmod g+w /etc/passwd + +# Add Tini +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN mv /tini /usr/local/bin/tini && chmod +x /usr/local/bin/tini + +# Install conda +USER $NB_UID +ARG PYTHON_VERSION=default +ENV MINICONDA_VERSION=4.8.3 \ + MINICONDA_MD5=751786b92c00b1aeae3f017b781018df \ + CONDA_VERSION=4.8.3 + +WORKDIR /tmp +RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \ + echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ + /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ + rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \ + echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \ + conda config --system --prepend channels conda-forge && \ + conda config --system --set auto_update_conda false && \ + conda config --system --set show_channel_urls true && \ + conda config --system --set channel_priority strict && \ + if [ ! $PYTHON_VERSION = 'default' ]; then conda install --yes python=$PYTHON_VERSION; fi && \ + conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \ + conda clean --all -f -y && \ + rm -rf /home/$NB_USER/.cache/yarn + +# Install latest sumbarine python sdk and notebook +RUN pip install notebook==6.1.3 && \ + git clone https://github.com/apache/submarine && \ + pip install submarine/submarine-sdk/pysubmarine && \ + cp submarine/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb $HOME && \ + cp -r submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json} $HOME && \ + rm submarine -rf + + + +EXPOSE $NB_PORT +ENTRYPOINT ["tini", "-g", "--"] +CMD ["start-notebook.sh"] +COPY --chown=${NB_USER}:users start-notebook.sh /usr/local/bin +WORKDIR ${HOME} diff --git a/dev-support/docker-images/jupyter/build.sh b/dev-support/docker-images/jupyter/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..e2b083ea3e842c393c260792c134b16e76e784d3 --- /dev/null +++ b/dev-support/docker-images/jupyter/build.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +JUPYTER_IMAGE="apache/submarine:jupyter-notebook-0.5.0" + +if [ -L ${BASH_SOURCE-$0} ]; then + PWD=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + PWD=$(dirname ${BASH_SOURCE-$0}) +fi +export CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd) +SUBMARINE_HOME=${CURRENT_PATH}/../../.. + +# build image +echo "Start building the ${JUPYTER_IMAGE} docker image ..." +cd ${CURRENT_PATH} +docker build -t ${JUPYTER_IMAGE} . diff --git a/dev-support/docker-images/jupyter/start-notebook.sh b/dev-support/docker-images/jupyter/start-notebook.sh new file mode 100755 index 0000000000000000000000000000000000000000..fce5e33981ac9f488d14b78aab70cb7849f3b36a --- /dev/null +++ b/dev-support/docker-images/jupyter/start-notebook.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# description: Start and stop daemon script for. +# + +set -euo pipefail + +# Install conda dependency +if [[ -n "${INSTALL_ENVIRONMENT_COMMAND:-}" ]]; then + /bin/bash -c "${INSTALL_ENVIRONMENT_COMMAND}" +fi + +NOTEBOOK_ARGS="--ip=0.0.0.0 --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password='' --NotebookApp.allow_origin='*'" +NB_USER="${NB_USER:-"jovyan"}" +NB_PREFIX="${NB_PREFIX:-"/"}" +NB_PORT="${NB_PORT:-8888}" + +if [[ -n "${NB_USER}" ]]; then + NOTEBOOK_ARGS="--notebook-dir=/home/${NB_USER} ${NOTEBOOK_ARGS}" +fi + +if [[ -n "${NB_PORT}" ]]; then + NOTEBOOK_ARGS="--port=${NB_PORT} ${NOTEBOOK_ARGS}" +fi + +if [[ -n "${NB_PREFIX}" ]]; then + NOTEBOOK_ARGS="--NotebookApp.base_url=${NB_PREFIX} ${NOTEBOOK_ARGS}" +fi + +/bin/bash -c "jupyter notebook ${NOTEBOOK_ARGS}" diff --git a/dev-support/docker-images/operator/Dockerfile b/dev-support/docker-images/operator/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d1a0080d24ed840828d00d0dca760420df92bdd5 --- /dev/null +++ b/dev-support/docker-images/operator/Dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.10 +MAINTAINER Apache Software Foundation + +# If you are in China, enabling the following two lines of code can speed up the build of the image, but it may cause failure in travis. +# So when submitting the code, please make sure the following 2 lines of code are in a commented state +# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/main" > /etc/apk/repositories +# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/community" >> /etc/apk/repositories + +RUN apk add tzdata --no-cache +ADD ./tmp/submarine-operator /usr/local/bin/submarine-operator + +ENTRYPOINT [ "/usr/local/bin/submarine-operator" ] diff --git a/dev-support/docker-images/operator/build.sh b/dev-support/docker-images/operator/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..5adbfe6bb7e78d1d0a99aa62658d45be32938469 --- /dev/null +++ b/dev-support/docker-images/operator/build.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +SUBMARINE_VERSION=0.5.0 +SUBMARINE_IMAGE_NAME="apache/submarine:operator-${SUBMARINE_VERSION}" + +if [ -L ${BASH_SOURCE-$0} ]; then + PWD=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + PWD=$(dirname ${BASH_SOURCE-$0}) +fi +export CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd) +export SUBMARINE_HOME=${CURRENT_PATH}/../../.. + +if [ ! -d "${SUBMARINE_HOME}/submarine-cloud/bin" ]; then + mkdir "${SUBMARINE_HOME}/submarine-cloud/bin" +fi +submarine_operator_exists=$(find -L "${SUBMARINE_HOME}/submarine-cloud/bin" -name "submarine-operator") +# Build source code if the package doesn't exist. +if [[ -z "${submarine_operator_exists}" ]]; then + cd "${SUBMARINE_HOME}/submarine-cloud" + mvn package +fi + +mkdir -p "${CURRENT_PATH}/tmp" +cp ${SUBMARINE_HOME}/submarine-cloud/bin/submarine-operator "${CURRENT_PATH}/tmp" + +# build image +cd ${CURRENT_PATH} +echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." +docker build -t ${SUBMARINE_IMAGE_NAME} . + +# clean temp file +rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/docker-images/submarine/Dockerfile b/dev-support/docker-images/submarine/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..792080ab9f49022da8eba67fbcb2e3a249850fb1 --- /dev/null +++ b/dev-support/docker-images/submarine/Dockerfile @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.10 +MAINTAINER Apache Software Foundation + +# If you are in China, enabling the following two lines of code can speed up the build of the image, but it may cause failure in travis. +# So when submitting the code, please make sure the following 2 lines of code are in a commented state +# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/main" > /etc/apk/repositories +# RUN echo "http://mirrors.ustc.edu.cn/alpine/v3.10/community" >> /etc/apk/repositories + +# INSTALL openjdk +RUN apk update && \ + apk add --no-cache openjdk8 tzdata bash tini && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo Asia/Shanghai > /etc/timezone && \ + apk del tzdata && \ + rm -rf /tmp/* /var/cache/apk/* + +ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk/jre + +# Install Submarine +ADD ./tmp/submarine-dist-*.tar.gz /opt/ +RUN ln -s /opt/submarine-dist-* "/opt/submarine-current" +ADD ./tmp/submarine-site.xml "/opt/submarine-current/conf/" +ADD ./tmp/submarine.sh "/opt/submarine-current/bin/" +ADD ./tmp/mysql-connector-java-5.1.39.jar "/opt/submarine-current/lib/" + +WORKDIR /opt/submarine-current + +# Submarine port +EXPOSE 8080 + +ENTRYPOINT ["/sbin/tini", "--"] + +CMD ["/bin/bash", "-c", "/opt/submarine-current/bin/submarine.sh start"] diff --git a/dev-support/docker-images/submarine/build.sh b/dev-support/docker-images/submarine/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..2fd38470b0d3def462481d84aef118de0cfe3d08 --- /dev/null +++ b/dev-support/docker-images/submarine/build.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +SUBMARINE_VERSION=0.5.0 +SUBMARINE_IMAGE_NAME="apache/submarine:server-${SUBMARINE_VERSION}" + +if [ -L ${BASH_SOURCE-$0} ]; then + PWD=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + PWD=$(dirname ${BASH_SOURCE-$0}) +fi +export CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd) +export SUBMARINE_HOME=${CURRENT_PATH}/../../.. + +if [ ! -d "${SUBMARINE_HOME}/submarine-dist/target" ]; then + mkdir "${SUBMARINE_HOME}/submarine-dist/target" +fi +submarine_dist_exists=$(find -L "${SUBMARINE_HOME}/submarine-dist/target" -name "submarine-dist-${SUBMARINE_VERSION}*.tar.gz") +# Build source code if the package doesn't exist. +if [[ -z "${submarine_dist_exists}" ]]; then + cd "${SUBMARINE_HOME}" + mvn clean package -DskipTests +fi + +mkdir -p "${CURRENT_PATH}/tmp" +cp ${SUBMARINE_HOME}/submarine-dist/target/submarine-dist-${SUBMARINE_VERSION}*.tar.gz "${CURRENT_PATH}/tmp" + +# download mysql connect java +MYSQL_VERSION=5.1.39 +MYSQL_JAR_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_VERSION}/mysql-connector-java-${MYSQL_VERSION}.jar" +tmpfile=$(mktemp) +trap "test -f $tmpfile && rm $tmpfile" RETURN +curl -L -o $tmpfile ${MYSQL_JAR_URL} +mv $tmpfile ${CURRENT_PATH}/tmp/mysql-connector-java-${MYSQL_VERSION}.jar + +# Replace the mysql jdbc.url in the submarine-site.xml file with the link name of the submarine container +# `submarine-database` is submarine database container name +cp ${SUBMARINE_HOME}/conf/submarine-site.xml "${CURRENT_PATH}/tmp/" +sed -i.bak 's/127.0.0.1:3306/submarine-database:3306/g' "${CURRENT_PATH}/tmp/submarine-site.xml" + +cp ${SUBMARINE_HOME}/bin/submarine.sh "${CURRENT_PATH}/tmp/" + +# build image +cd ${CURRENT_PATH} +echo "Start building the ${SUBMARINE_IMAGE_NAME} docker image ..." +docker build -t ${SUBMARINE_IMAGE_NAME} . + +# clean temp file +rm -rf "${CURRENT_PATH}/tmp" diff --git a/dev-support/k8s/deploy-kubeflow-operators.sh b/dev-support/k8s/deploy-kubeflow-operators.sh new file mode 100755 index 0000000000000000000000000000000000000000..ec12aa65a12cc07dab16604250c5f2910dec85a7 --- /dev/null +++ b/dev-support/k8s/deploy-kubeflow-operators.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +readonly TF_OPERATOR_IMAGE="apache/submarine:tf_operator-v1.1.0-g92389064" +readonly PYTORCH_OPERATOR_IMAGE="apache/submarine:pytorch-operator-v1.1.0-gd596e904" +readonly TF_MNIST_IMAGE="apache/submarine:tf-mnist-with-summaries-1.0" +readonly PT_MNIST_IMAGE="apache/submarine:pytorch-dist-mnist-1.0" + +if [ -L "${BASH_SOURCE-$0}" ]; then + PWD=$(dirname "$(readlink "${BASH_SOURCE-$0}")") +else + PWD=$(dirname "${BASH_SOURCE-$0}") +fi +CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd) +export CURRENT_PATH +export SUBMARINE_HOME=${CURRENT_PATH}/../.. +# lib.sh use the ROOT variable +export ROOT="${SUBMARINE_HOME}/submarine-cloud/" +export KUBECONFIG="${HOME}/.kube/kind-config-${clusterName:-kind}" + +# shellcheck source=./../../submarine-cloud/hack/lib.sh +source "${SUBMARINE_HOME}/submarine-cloud/hack/lib.sh" + +########################################### +# Load local docker image into registry +# Globals: +# KIND_BIN +# Arguments: +# image +########################################### +function load_image_to_registry() { + if [[ ! $(docker inspect "$1" > /dev/null) ]] ; then + docker pull "$1" + fi + ${KIND_BIN} load docker-image "$1" +} + +########################################### +# Deploy tf-operator on K8s +# Globals: +# KUBECTL_BIN +# CURRENT_PATH +# TF_OPERATOR_IMAGE +# Arguments: +# useSample +########################################### +function deploy_tf_operator() { + load_image_to_registry "${TF_OPERATOR_IMAGE}" + + ${KUBECTL_BIN} apply -f "${CURRENT_PATH}/tfjob/crd.yaml" + ${KUBECTL_BIN} kustomize "${CURRENT_PATH}/tfjob/operator" \ + | ${KUBECTL_BIN} apply -f - + + if [[ ${1:-} == "true" ]]; then + load_image_to_registry "${TF_MNIST_IMAGE}" + fi +} + +########################################### +# Deploy tf-operator on K8s +# Globals: +# KUBECTL_BIN +# CURRENT_PATH +# PYTORCH_OPERATOR_IMAGE +# Arguments: +# useSample +########################################### +function deploy_pytorch_operator() { + load_image_to_registry "${PYTORCH_OPERATOR_IMAGE}" + ${KUBECTL_BIN} apply -f "${CURRENT_PATH}/pytorchjob" + + if [[ ${1:-} == "true" ]]; then + load_image_to_registry "${PT_MNIST_IMAGE}" + fi +} + +########################################### +# Print the usage information +########################################### +function usage() { + cat </dev/null; pwd) +export CURRENT_PATH +export SUBMARINE_HOME=${CURRENT_PATH}/../.. +# lib.sh use the ROOT variable +export ROOT="${SUBMARINE_HOME}/submarine-cloud/" +export KUBECONFIG="${HOME}/.kube/kind-config-${clusterName:-kind}" + +# shellcheck source=./../../submarine-cloud/hack/lib.sh +source "${SUBMARINE_HOME}/submarine-cloud/hack/lib.sh" + +########################################### +# Load local docker image into registry +# Globals: +# KIND_BIN +# Arguments: +# image +########################################### +function load_image_to_registry() { + if [[ ! $(docker inspect "$1" > /dev/null) ]] ; then + docker pull "$1" + fi + ${KIND_BIN} load docker-image "$1" +} + + +function main() { + + hack::ensure_kubectl + + load_image_to_registry "${NOTEBOOK_CONTROLLER_IMAGE}" + ${KUBECTL_BIN} apply -k "${CURRENT_PATH}/notebook-controller" + +} + +main "$@" diff --git a/dev-support/k8s/deploy-traefik.sh b/dev-support/k8s/deploy-traefik.sh new file mode 100755 index 0000000000000000000000000000000000000000..3fb84003465b1aa58cbb464c33011a8b1a33f219 --- /dev/null +++ b/dev-support/k8s/deploy-traefik.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +readonly TRAEFIK_IMAGE="traefik:v2.2" + +if [ -L "${BASH_SOURCE-$0}" ]; then + PWD=$(dirname "$(readlink "${BASH_SOURCE-$0}")") +else + PWD=$(dirname "${BASH_SOURCE-$0}") +fi +CURRENT_PATH=$(cd "${PWD}">/dev/null; pwd) +export CURRENT_PATH +export SUBMARINE_HOME=${CURRENT_PATH}/../.. +# lib.sh use the ROOT variable +export ROOT="${SUBMARINE_HOME}/submarine-cloud/" +export KUBECONFIG="${HOME}/.kube/kind-config-${clusterName:-kind}" + +# shellcheck source=./../../submarine-cloud/hack/lib.sh +source "${SUBMARINE_HOME}/submarine-cloud/hack/lib.sh" + +########################################### +# Load local docker image into registry +# Globals: +# KIND_BIN +# Arguments: +# image +########################################### +function load_image_to_registry() { + if [[ ! $(docker inspect "$1" > /dev/null) ]] ; then + docker pull "$1" + fi + ${KIND_BIN} load docker-image "$1" +} + +function main() { + hack::ensure_kubectl + echo "Setting up ingress on a kind cluster." + load_image_to_registry "${TRAEFIK_IMAGE}" + ${KUBECTL_BIN} apply -k "${CURRENT_PATH}/traefik" +} + +main "@" diff --git a/dev-support/k8s/notebook-controller/cluster-role-binding.yaml b/dev-support/k8s/notebook-controller/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1a394540145b946de5afb10e71c63c65d4c77a4 --- /dev/null +++ b/dev-support/k8s/notebook-controller/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: role +subjects: +- kind: ServiceAccount + name: service-account diff --git a/dev-support/k8s/notebook-controller/cluster-role.yaml b/dev-support/k8s/notebook-controller/cluster-role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16b6253a8f6b3b5bc551950501c5ebe555a5ccd1 --- /dev/null +++ b/dev-support/k8s/notebook-controller/cluster-role.yaml @@ -0,0 +1,107 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: role +rules: +- apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - '*' +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - kubeflow.org + resources: + - notebooks + - notebooks/status + - notebooks/finalizers + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - virtualservices + verbs: + - '*' + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-notebooks-admin + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true" +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-notebooks-admin: "true" +rules: [] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-notebooks-edit + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-notebooks-admin: "true" +rules: +- apiGroups: + - kubeflow.org + resources: + - notebooks + - notebooks/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-notebooks-view + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" +rules: +- apiGroups: + - kubeflow.org + resources: + - notebooks + - notebooks/status + verbs: + - get + - list + - watch diff --git a/dev-support/k8s/notebook-controller/crd.yaml b/dev-support/k8s/notebook-controller/crd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6556bd4ccbe0c3afb189264d4402c2ac00861f6 --- /dev/null +++ b/dev-support/k8s/notebook-controller/crd.yaml @@ -0,0 +1,64 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: notebooks.kubeflow.org +spec: + group: kubeflow.org + names: + kind: Notebook + plural: notebooks + singular: notebook + scope: Namespaced + subresources: + status: {} + versions: + - name: v1alpha1 + served: true + storage: false + - name: v1beta1 + served: true + storage: true + - name: v1 + served: true + storage: false + validation: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + template: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + properties: + spec: + type: object + type: object + type: object + status: + properties: + conditions: + description: Conditions is an array of current conditions + items: + properties: + type: + description: Type of the confition/ + type: string + required: + - type + type: object + type: array + required: + - conditions + type: object diff --git a/dev-support/k8s/notebook-controller/deployment.yaml b/dev-support/k8s/notebook-controller/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb7c93c641353d3763f751b67f3003d57866667f --- /dev/null +++ b/dev-support/k8s/notebook-controller/deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: manager + image: apache/submarine:notebook-controller-v1.1.0-g253890cb + command: + - /manager + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /metrics + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 30 + serviceAccountName: service-account diff --git a/dev-support/k8s/notebook-controller/deployment_patch.yaml b/dev-support/k8s/notebook-controller/deployment_patch.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7dfb43349b91b8150eefe5b90be904fb5938fe7 --- /dev/null +++ b/dev-support/k8s/notebook-controller/deployment_patch.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment +spec: + template: + spec: + containers: + - name: manager + env: + # We use a patch to set the USE_ISTIO because in other patches + # we want to set it to a configMapRef and so if we include the value + # in the base when we do the merge we end up with 2 fields setting the value. + - name: USE_ISTIO + value: "false" \ No newline at end of file diff --git a/dev-support/k8s/notebook-controller/kustomization.yaml b/dev-support/k8s/notebook-controller/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0c75d5827d92896c845aa7684095240e1eae1b2 --- /dev/null +++ b/dev-support/k8s/notebook-controller/kustomization.yaml @@ -0,0 +1,42 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role-binding.yaml +- cluster-role.yaml +- crd.yaml +- deployment.yaml +- service-account.yaml +- service.yaml +namePrefix: notebook-controller- +namespace: default +patchesStrategicMerge: +- deployment_patch.yaml +commonLabels: + app: notebook-controller + kustomize.component: notebook-controller +images: +- name: apache/submarine + newName: apache/submarine + newTag: notebook-controller-v1.1.0-g253890cb +configMapGenerator: +- name: parameters + literals: + - USE_ISTIO=false + - ISTIO_GATEWAY= +generatorOptions: + disableNameSuffixHash: true +vars: +- fieldref: + fieldPath: data.USE_ISTIO + name: USE_ISTIO + objref: + apiVersion: v1 + kind: ConfigMap + name: parameters +- fieldref: + fieldPath: data.ISTIO_GATEWAY + name: ISTIO_GATEWAY + objref: + apiVersion: v1 + kind: ConfigMap + name: parameters diff --git a/dev-support/k8s/notebook-controller/service-account.yaml b/dev-support/k8s/notebook-controller/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a36cbd800f609a441b72e0f38db6a5cdeea4a91f --- /dev/null +++ b/dev-support/k8s/notebook-controller/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: service-account diff --git a/dev-support/k8s/notebook-controller/service.yaml b/dev-support/k8s/notebook-controller/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7368f970344b6b76d7acaa9af79f36d8d1fadea --- /dev/null +++ b/dev-support/k8s/notebook-controller/service.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: service +spec: + ports: + - port: 443 diff --git a/dev-support/k8s/pytorchjob/crd.yaml b/dev-support/k8s/pytorchjob/crd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a8cf899d92b240de5adfdb5967d4534bb3bb682 --- /dev/null +++ b/dev-support/k8s/pytorchjob/crd.yaml @@ -0,0 +1,42 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pytorchjobs.kubeflow.org +spec: + additionalPrinterColumns: + - JSONPath: .status.conditions[-1:].type + name: State + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: kubeflow.org + names: + kind: PyTorchJob + plural: pytorchjobs + singular: pytorchjob + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + pytorchReplicaSpecs: + properties: + Master: + properties: + replicas: + maximum: 1 + minimum: 1 + type: integer + Worker: + properties: + replicas: + minimum: 1 + type: integer + versions: + - name: v1 + served: true + storage: true diff --git a/dev-support/k8s/pytorchjob/deploy-pytorch-operator.sh b/dev-support/k8s/pytorchjob/deploy-pytorch-operator.sh new file mode 100755 index 0000000000000000000000000000000000000000..07f7b075faa52411e24542944d3eb4f3eb2c3ebf --- /dev/null +++ b/dev-support/k8s/pytorchjob/deploy-pytorch-operator.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# description: Start and stop daemon script for. +# + +kubectl apply -f ./namespace.yaml +sleep 5 +# there're two deployment yaml file. One for K8s version prior to 1.16. One for 1.16+ due to API incompatibility +kubectl apply -f ./ diff --git a/dev-support/k8s/pytorchjob/deployment.yaml b/dev-support/k8s/pytorchjob/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ebcea736e2540917059bcdae2ad1e3f009af72f --- /dev/null +++ b/dev-support/k8s/pytorchjob/deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: extensions/v1beta1 +#apiVersion: apps/v1 +kind: Deployment +metadata: + name: pytorch-operator + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + name: pytorch-operator + template: + metadata: + labels: + name: pytorch-operator + spec: + containers: + - command: + - /pytorch-operator.v1 + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: apache/submarine:pytorch-operator-v1.1.0-gd596e904 + name: pytorch-operator + serviceAccountName: pytorch-operator diff --git a/dev-support/k8s/pytorchjob/deployment_v1.16plus.yaml b/dev-support/k8s/pytorchjob/deployment_v1.16plus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5db665bd5fe8ff6aae46294e65f295a0473072e1 --- /dev/null +++ b/dev-support/k8s/pytorchjob/deployment_v1.16plus.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + name: pytorch-operator + name: pytorch-operator + namespace: default +spec: + progressDeadlineSeconds: 2147483647 + replicas: 1 + revisionHistoryLimit: 2147483647 + selector: + matchLabels: + name: pytorch-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + name: pytorch-operator + spec: + containers: + - command: + - /pytorch-operator.v1 + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: apache/submarine:pytorch-operator-v1.1.0-gd596e904 + imagePullPolicy: IfNotPresent + name: pytorch-operator + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: pytorch-operator + serviceAccountName: pytorch-operator + terminationGracePeriodSeconds: 30 +status: {} diff --git a/dev-support/k8s/pytorchjob/namespace.yaml b/dev-support/k8s/pytorchjob/namespace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..559fbc3276c8b8d8767b7cfc243f83d22a79ef5f --- /dev/null +++ b/dev-support/k8s/pytorchjob/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: submarine diff --git a/dev-support/k8s/pytorchjob/podgroup.yaml b/dev-support/k8s/pytorchjob/podgroup.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1432fc160f238e8b90aa6e7cb3fdc4f29f2b1bb4 --- /dev/null +++ b/dev-support/k8s/pytorchjob/podgroup.yaml @@ -0,0 +1,39 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: podgroups.scheduling.incubator.k8s.io +spec: + group: scheduling.incubator.k8s.io + names: + kind: PodGroup + plural: podgroups + scope: Namespaced + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + minMember: + format: int32 + type: integer + type: object + status: + properties: + succeeded: + format: int32 + type: integer + failed: + format: int32 + type: integer + running: + format: int32 + type: integer + type: object + type: object + version: v1alpha1 diff --git a/dev-support/k8s/pytorchjob/rbac.yaml b/dev-support/k8s/pytorchjob/rbac.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0261c8501c37c9fe20309ad30e69336efbdd7ecb --- /dev/null +++ b/dev-support/k8s/pytorchjob/rbac.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: pytorch-operator + name: pytorch-operator + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + app: pytorch-operator + name: pytorch-operator +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - events + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + app: pytorch-operator + name: pytorch-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pytorch-operator +subjects: +- kind: ServiceAccount + name: pytorch-operator + namespace: default +--- diff --git a/dev-support/k8s/pytorchjob/service.yaml b/dev-support/k8s/pytorchjob/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e57a496ff80f8c538321b04e2c37f072217cb4b --- /dev/null +++ b/dev-support/k8s/pytorchjob/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "8443" + prometheus.io/scrape: "true" + labels: + app: pytorch-operator + name: pytorch-operator + namespace: default +spec: + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: + name: pytorch-operator + type: ClusterIP diff --git a/dev-support/k8s/tfjob/crd.yaml b/dev-support/k8s/tfjob/crd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38f850bd7b1dbb604a7edee87ba16cc2584a359e --- /dev/null +++ b/dev-support/k8s/tfjob/crd.yaml @@ -0,0 +1,42 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tfjobs.kubeflow.org +spec: + group: kubeflow.org + scope: Namespaced + names: + kind: TFJob + singular: tfjob + plural: tfjobs + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + tfReplicaSpecs: + properties: + # The validation works when the configuration contains + # `Worker`, `PS` or `Chief`. Otherwise it will not be validated. + Worker: + properties: + replicas: + type: integer + minimum: 1 + PS: + properties: + replicas: + type: integer + minimum: 1 + Chief: + properties: + replicas: + type: integer + minimum: 1 + maximum: 1 diff --git a/dev-support/k8s/tfjob/operator/cluster-role-binding.yaml b/dev-support/k8s/tfjob/operator/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e05aad7fc47a63b548e534f1a6efc1c4eb56474f --- /dev/null +++ b/dev-support/k8s/tfjob/operator/cluster-role-binding.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + app: tf-job-operator + name: tf-job-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tf-job-operator +subjects: +- kind: ServiceAccount + name: tf-job-operator diff --git a/dev-support/k8s/tfjob/operator/cluster-role.yaml b/dev-support/k8s/tfjob/operator/cluster-role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2740b98e0b27e2e5168393d966abebcaf372803d --- /dev/null +++ b/dev-support/k8s/tfjob/operator/cluster-role.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + app: tf-job-operator + name: tf-job-operator +rules: + - apiGroups: + - kubeflow.org + resources: + - tfjobs + - tfjobs/status + - tfjobs/finalizers + verbs: + - '*' + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + - services + - endpoints + - events + verbs: + - '*' + - apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - '*' + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-tfjobs-admin + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true" +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-tfjobs-admin: "true" +rules: [] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-tfjobs-edit + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-tfjobs-admin: "true" +rules: + - apiGroups: + - kubeflow.org + resources: + - tfjobs + - tfjobs/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-tfjobs-view + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" +rules: + - apiGroups: + - kubeflow.org + resources: + - tfjobs + - tfjobs/status + verbs: + - get + - list + - watch diff --git a/dev-support/k8s/tfjob/operator/deployment.yaml b/dev-support/k8s/tfjob/operator/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4115b9100dbd33ca5316a85fa85b44d14569e5e --- /dev/null +++ b/dev-support/k8s/tfjob/operator/deployment.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tf-job-operator +spec: + replicas: 1 + template: + metadata: + labels: + name: tf-job-operator + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - args: + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: apache/submarine:tf_operator-v1.1.0-g92389064 + name: tf-job-operator + serviceAccountName: tf-job-operator diff --git a/dev-support/k8s/tfjob/operator/kustomization.yaml b/dev-support/k8s/tfjob/operator/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06f3f6ee064fcd6233df3d9022e44f467d578b84 --- /dev/null +++ b/dev-support/k8s/tfjob/operator/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- cluster-role-binding.yaml +- cluster-role.yaml +- deployment.yaml +- service-account.yaml +- service.yaml +commonLabels: + kustomize.component: tf-job-operator +images: +- name: apache/submarine + newName: apache/submarine + newTag: tf_operator-v1.1.0-g92389064 diff --git a/dev-support/k8s/tfjob/operator/service-account.yaml b/dev-support/k8s/tfjob/operator/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7be4e33e68795aa298db796091eb0fbbcfb36b6 --- /dev/null +++ b/dev-support/k8s/tfjob/operator/service-account.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: tf-job-dashboard + name: tf-job-dashboard +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: tf-job-operator + name: tf-job-operator diff --git a/dev-support/k8s/tfjob/operator/service.yaml b/dev-support/k8s/tfjob/operator/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97f92e3ea185b7e19922f3812f56e02b995e7847 --- /dev/null +++ b/dev-support/k8s/tfjob/operator/service.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/scrape: "true" + prometheus.io/port: "8443" + labels: + app: tf-job-operator + name: tf-job-operator +spec: + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: + name: tf-job-operator + type: ClusterIP diff --git a/dev-support/k8s/traefik/cluster-role-binding.yaml b/dev-support/k8s/traefik/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe862602be23e58ebc9581ad450d9e3ee7d1ab31 --- /dev/null +++ b/dev-support/k8s/traefik/cluster-role-binding.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: traefik-ingress-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: traefik-ingress-controller +subjects: +- kind: ServiceAccount + name: traefik-ingress-controller + namespace: default diff --git a/dev-support/k8s/traefik/cluster-role.yaml b/dev-support/k8s/traefik/cluster-role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74537deaa091c08972073aa415a8176de18dcee6 --- /dev/null +++ b/dev-support/k8s/traefik/cluster-role.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: traefik-ingress-controller + +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - traefik.containo.us + resources: + - middlewares + - ingressroutes + - traefikservices + - ingressroutetcps + - ingressrouteudps + - tlsoptions + - tlsstores + verbs: + - get + - list + - watch diff --git a/dev-support/k8s/traefik/crd.yaml b/dev-support/k8s/traefik/crd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76776bdec11e8312955463bfd8d1b6e00ea40325 --- /dev/null +++ b/dev-support/k8s/traefik/crd.yaml @@ -0,0 +1,120 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutes.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRoute + plural: ingressroutes + singular: ingressroute + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: middlewares.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: Middleware + plural: middlewares + singular: middleware + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutetcps.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRouteTCP + plural: ingressroutetcps + singular: ingressroutetcp + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressrouteudps.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRouteUDP + plural: ingressrouteudps + singular: ingressrouteudp + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tlsoptions.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TLSOption + plural: tlsoptions + singular: tlsoption + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tlsstores.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TLSStore + plural: tlsstores + singular: tlsstore + scope: Namespaced + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: traefikservices.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TraefikService + plural: traefikservices + singular: traefikservice + scope: Namespaced diff --git a/dev-support/k8s/traefik/deployment.yaml b/dev-support/k8s/traefik/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be670cba1bfb3c3c9af90b4bbf1973c917572cb0 --- /dev/null +++ b/dev-support/k8s/traefik/deployment.yaml @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: default + name: traefik + labels: + app: traefik +spec: + replicas: 1 + selector: + matchLabels: + app: traefik + template: + metadata: + labels: + app: traefik + spec: + serviceAccountName: traefik-ingress-controller + containers: + - name: traefik + image: traefik:v2.2 + args: + - --accesslog + - --entrypoints.web.Address=:80 + - --entrypoints.websecure.Address=:443 + - --ping=true + - --providers.kubernetescrd + - --providers.kubernetesingress + ports: + - name: web + containerPort: 80 + - name: websecure + containerPort: 443 diff --git a/dev-support/k8s/traefik/kustomization.yaml b/dev-support/k8s/traefik/kustomization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7f05d060e2cdf8dd3a634d520936ea5e683ac94 --- /dev/null +++ b/dev-support/k8s/traefik/kustomization.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- cluster-role.yaml +- cluster-role-binding.yaml +- crd.yaml +- deployment.yaml +- service-account.yaml +- service.yaml +namespace: default +commonLabels: + kustomize.component: traefik + +images: +- name: traefik + newName: traefik + newTag: v2.2 diff --git a/dev-support/k8s/traefik/service-account.yaml b/dev-support/k8s/traefik/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fee03de6233593e7ad87de92e2a2ba815f34f6b2 --- /dev/null +++ b/dev-support/k8s/traefik/service-account.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: traefik-ingress-controller diff --git a/dev-support/k8s/traefik/service.yaml b/dev-support/k8s/traefik/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d00005ae433433b90670ee0cea76a2ece995ff33 --- /dev/null +++ b/dev-support/k8s/traefik/service.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: traefik +spec: + type: NodePort + ports: + - protocol: TCP + name: web + port: 80 + nodePort: 32080 + - protocol: TCP + name: websecure + port: 443 + nodePort: 32443 + selector: + app: traefik diff --git a/dev-support/maven-config/checkstyle.xml b/dev-support/maven-config/checkstyle.xml new file mode 100644 index 0000000000000000000000000000000000000000..e31aabf00c6fb44a1979c36b94f39e6fdedef54e --- /dev/null +++ b/dev-support/maven-config/checkstyle.xml @@ -0,0 +1,289 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dev-support/maven-config/scalastyle-config.xml b/dev-support/maven-config/scalastyle-config.xml new file mode 100644 index 0000000000000000000000000000000000000000..0deecf9cfd7bd6e23074ec8b9f5b3a760f00bd8a --- /dev/null +++ b/dev-support/maven-config/scalastyle-config.xml @@ -0,0 +1,288 @@ + + + + Scalastyle standard configuration + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + + ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + + + + Runtime\.getRuntime\.addShutdownHook + + + + + mutable\.SynchronizedBuffer + + + + + (\.toUpperCase|\.toLowerCase)(?!(\(|\(Locale.ROOT\))) + + + + + throw new \w+Error\( + + + + + JavaConversions + Instead of importing implicits in scala.collection.JavaConversions._, import + scala.collection.JavaConverters._ and use .asScala / .asJava methods + + + + org\.apache\.commons\.lang\. + Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead + of Commons Lang 2 (package org.apache.commons.lang.*) + + + + extractOpt + Use jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter + is slower. + + + + + java,scala,3rdParty,submarine + javax?\..* + scala\..* + (?!org\.apache\.submarine\.).* + org\.apache\.submarine\..* + false + + + + + + COMMA + + + + + \)\{ + + + + + (?m)^(\s*)/[*][*].*$(\r|)\n^\1 [*] + Use Javadoc style indentation for multiline comments + + + + case[^\n>]*=>\s*\{ + Omit braces in case clauses. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 800> + + + + + 30 + + + + + 10 + + + + + 50 + + + + + + + + + + + -1,0,1,2,3 + + + diff --git a/dev-support/mini-submarine/Dockerfile b/dev-support/mini-submarine/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..80e066b01440d00979860e2d2956f6d38ea9d7e3 --- /dev/null +++ b/dev-support/mini-submarine/Dockerfile @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 +#INSTALL JAVA +RUN apt-get -q update \ + && apt-get install -y wget software-properties-common \ + && add-apt-repository -y ppa:webupd8team/java \ + && apt-get update \ + && wget http://archive.ubuntu.com/ubuntu/pool/main/m/mesa/libgl1-mesa-dri_19.2.8-0ubuntu0~18.04.2_amd64.deb \ + && wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu60_60.2-3ubuntu3.1_amd64.deb \ + && apt-get install -y ./libgl1-mesa-dri_19.2.8-0ubuntu0~18.04.2_amd64.deb \ + && apt-get install -y ./libicu60_60.2-3ubuntu3.1_amd64.deb \ + && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/jre + +# INSTALL Docker +RUN \ + apt-get update && \ + apt-get -y install apt-transport-https ca-certificates curl software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + apt-get update && \ + apt-get -y install docker-ce +# So no need to mount host's /var/run/docker.sock, dockerd will create in local fs +VOLUME /var/lib/docker + +#INSTALL user tools +RUN \ + apt-get update && \ + apt-get -y install vim + +# Install libgomp1 for MXNet +RUN \ + apt-get update && \ + apt-get -y install libgomp1 + +#INSTALL HADOOP +# Add native libs +ARG HADOOP_VERSION= +ADD hadoop-${HADOOP_VERSION}.tar.gz /usr/local +#ADD hadoop-native-${HADOOP_VERSION}.tar /usr/local/hadoop-${HADOOP_VERSION}/lib/native + +ENV HADOOP_PREFIX=/usr/local/hadoop \ + HADOOP_COMMON_HOME=/usr/local/hadoop \ + HADOOP_HDFS_HOME=/usr/local/hadoop \ + HADOOP_MAPRED_HOME=/usr/local/hadoop \ + HADOOP_YARN_HOME=/usr/local/hadoop \ + HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop \ + YARN_CONF_DIR=/usr/local/hadoop/etc/hadoop \ + PATH=${PATH}:/usr/local/hadoop/bin + +RUN \ + cd /usr/local && mv ./hadoop-${HADOOP_VERSION} hadoop && \ + rm -f ${HADOOP_PREFIX}/logs/* + +ARG ZK_VERSION=3.4.14 +ADD zookeeper-${ZK_VERSION}.tar.gz /usr/local +RUN mv /usr/local/zookeeper-${ZK_VERSION} /usr/local/zookeeper +RUN sed "s#/tmp/zookeeper#/tmp/staging/zookeeper#" /usr/local/zookeeper/conf/zoo_sample.cfg > /usr/local/zookeeper/conf/zoo.cfg + +WORKDIR $HADOOP_PREFIX + +# Hdfs ports +EXPOSE 50010 50020 50070 50075 50090 8020 9000 +# Mapred ports +EXPOSE 19888 +#Yarn ports +EXPOSE 8030 8031 8032 8033 8040 8042 8088 +# ZK ports +EXPOSE 2181 2888 3888 +#Other ports +EXPOSE 49707 2122 +# Workbench port +EXPOSE 8080 + +#Add spark dynamic allocation jar +#ADD spark-2.4.0-yarn-shuffle.jar /usr/local/hadoop/share/hadoop/yarn/spark-2.4.0-yarn-shuffle.jar + +# Create users +RUN \ + groupadd -g 1007 hadoop && \ + useradd -m -G hadoop -u 1008 -s /bin/bash yarn && \ + chown -R root:hadoop /usr/local/hadoop && \ + chown -R yarn:hadoop /usr/local/zookeeper + +# Copy Config +COPY conf /tmp/hadoop-config + +ENV HADOOP_VER=${HADOOP_VERSION} + +#Install Spark +ARG SPARK_VERSION= +ENV SPARK_VER=${SPARK_VERSION} +ADD spark-${SPARK_VERSION}-bin-hadoop2.7.tgz /opt +ADD spark-defaults-dynamic-allocation.conf /opt/spark-${SPARK_VERSION}/conf/spark-defaults.conf +RUN apt-get update && apt-get install -y vim python python-numpy wget zip python3 python3-distutils + +# Add pyspark sample +ADD spark-script /home/yarn/spark-script +RUN chown -R yarn /home/yarn/spark-script && \ + chmod +x -R /home/yarn/spark-script + +# Add distributedShell example +ADD conf/yarn-ds-docker.sh /home/yarn +RUN chown -R yarn /home/yarn/yarn-ds-docker.sh && \ + chmod +x /home/yarn/yarn-ds-docker.sh + +# set image name env +ARG IMAGE_NAME= +ENV IMAGE_N=${IMAGE_NAME} + +# Install Submarine +ARG SUBMARINE_VERSION= +ENV SUBMARINE_VER=${SUBMARINE_VERSION} +ADD submarine-dist-${SUBMARINE_VER}-hadoop*.tar.gz /opt +ADD pysubmarine /opt/pysubmarine +RUN ln -s /opt/submarine-dist-${SUBMARINE_VER}* /opt/submarine-current +ADD submarine /home/yarn/submarine +ADD database /home/yarn/database + +# Build virtual python env +RUN cd /home/yarn/submarine && \ + chmod +x /home/yarn/submarine/* && \ + /home/yarn/submarine/build_python_virtual_env.sh + +# Grant read permission for submarine job +RUN chown -R yarn /home/yarn/submarine +RUN chown -R yarn /opt/* diff --git a/dev-support/mini-submarine/README.md b/dev-support/mini-submarine/README.md new file mode 100644 index 0000000000000000000000000000000000000000..063bbee0576761d9b076cc8d71f5f5fd31ea9df8 --- /dev/null +++ b/dev-support/mini-submarine/README.md @@ -0,0 +1,337 @@ + + +# Mini-submarine + +This is a docker image built for submarine development and quick start test. + +**Please Note: don't use the image in production environment. It's only for test purpose.** + +## Start mini-submarine + +### Use the image we provide + +``` +docker pull apache/submarine:mini-0.5.0 +``` + +### Create image by yourself + +> You may need a VPN if your network is limited + +1.Clone the source code of Submarine +``` +git clone https://github.com/apache/submarine.git +``` + +2.Build Submarine +``` +cd ./submarine +mvn clean package -DskipTests +``` + +3.Build image of mini-submarine +> You can download in advance of these three kind of compressed file for building : +> zookeeper-3.4.14.tar.gz , hadoop-2.9.2.tar.gz , spark-2.4.4-bin-hadoop2.7.tgz +> and put them into "submarine/dev-support/mini-submarine/" + +``` +cd submarine/dev-support/mini-submarine/ +./build_mini-submarine.sh +``` +#### Package An Existing Release Candidates +When doing release, the release manager might needs to package a artifact candidates in this docker image and public the image candidate for a vote. +In this scenario, we can do this: + +Put submarine candidate artifacts to a folder like "~/releases/submarine-release" +``` +$ ls $release_candidates_path +submarine-dist-0.5.0-hadoop-2.9.tar.gz submarine-dist-0.5.0-src.tar.gz.asc +submarine-dist-0.5.0-hadoop-2.9.tar.gz.asc submarine-dist-0.5.0-src.tar.gz.sha512 +submarine-dist-0.5.0-hadoop-2.9.tar.gz.sha512 submarine-dist-0.5.0-src.tar.gz +``` +``` +export submarine_version=0.5.0 +export release_candidates_path=~/releases/submarine-release +./build_mini-submarine.sh +#docker run -it -h submarine-dev --net=bridge --privileged -P local/mini-submarine:0.5.0 /bin/bash +docker tag local/mini-submarine:0.5.0 apache/mini-submarine:0.5.0:RC0 +docker push apache/mini-submarine:0.5.0:RC0 +``` +In the container, we can verify that the submarine jar version is the expected 0.5.0. Then we can upload this image with a "RC" tag for a vote. + +### Run mini-submarine image + +``` +docker run -it -h submarine-dev --name mini-submarine --net=bridge --privileged -P local/mini-submarine:0.5.0 /bin/bash + +# In the container, use root user to bootstrap hdfs and yarn +/tmp/hadoop-config/bootstrap.sh + +# Two commands to check if yarn and hdfs is running as expected +yarn node -list -showDetails +``` + +If you pull the image directly, please replace "local/mini-submarine:0.5.0" with "apache/submarine:mini-0.5.0". + +### You should see info like this: + +``` +Total Nodes:1 + Node-Id Node-State Node-Http-Address Number-of-Running-Containers +submarine-dev:35949 RUNNING submarine-dev:8042 0 +Detailed Node Information : + Configured Resources : + Allocated Resources : + Resource Utilization by Node : PMem:4144 MB, VMem:4189 MB, VCores:0.25308025 + Resource Utilization by Containers : PMem:0 MB, VMem:0 MB, VCores:0.0 + Node-Labels : +``` + +``` +hdfs dfs -ls /user +``` + +> drwxr-xr-x - yarn supergroup 0 2019-07-22 07:59 /user/yarn + +## Run workbench server + +1. Setup mysql mariadb server +> Because mysql and mariadb use the GPL license, So there is no binary file containing mysql in the image, you need to manually execute the script to install it. + +``` +/tmp/hadoop-config/setup-mysql.sh +``` + +You can execute command `mysql -uroot` login mysql mariadb. + +2. Start submarine server +``` +su yarn +/opt/submarine-current/bin/submarine-daemon.sh start getMysqlJar +``` + +3. Login submarine workbench + +Execute the following command in your host machine, Get the access URL of the submarine workbench running in docker + +```shell +echo "http://localhost:$(docker inspect --format='{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}' mini-submarine)" +``` +The URL returned by the command (like to: http://localhost:32819) is opened through a browser. The username and initial password of the workbench are both `admin`. + +## Run a submarine job + +### Switch to user yarn + +``` +su yarn +``` + +### Navigate to submarine example directory +``` +cd /home/yarn/submarine/ +``` + +### Run a mnist TF job with submarine + TonY runtime +``` +# run TF 1 distributed training job +./run_submarine_mnist_tony.sh + +# run TF 2 distributed training job +./run_submarine_mnist_tf2_tony.sh +``` +When run_submarine_mnist_tony.sh is executed, mnist data is download from the url, [google mnist](https://storage.googleapis.com/cvdf-datasets/mnist/), by default. If the url is unaccessible, you can use parameter "-d" to specify a customized url. +For example, if you are in mainland China, you can use the following command +``` +./run_submarine_mnist_tony.sh -d http://yann.lecun.com/exdb/mnist/ +``` + +### Run a mnist TF job via submarine server + +Submarine server is supposed to manage jobs lifecycle. Clients can just submit +job parameters or yaml file to submarine server instead of submitting jobs +directly by themselves. Submarine server can handle the rest of the work. + +Set submarine.server.rpc.enabled to true in the file of +/opt/submarine-current/conf/submarine-site +``` + + submarine.server.rpc.enabled + true + Run jobs using rpc server. + +``` +Run the following command to submit a job via submarine server +``` +./run_submarine_mnist_tony_rpc.sh +``` + +### Try your own submarine program + +Run container with your source code. You can also use "docker cp" to an existing running container + +1. `docker run -it -h submarine-dev --net=bridge --privileged -v pathToMyScrit.py:/home/yarn/submarine/myScript.py local/hadoop-docker:submarine /bin/bash` + +2. Refer to the `run_submarine_mnist_tony.sh` and modify the script to your script + +3. Try to run it. Since this is a single node environment, keep in mind that the workers could have conflicts with each other. For instance, the mnist_distributed.py example has a workaround to fix the conflicts when two workers are using same "data_dir" to download data set. + + +## Update Submarine Version + +You can follow the documentation instructions to update your own modified and compiled submarine package to the submarine container. + +### Build Submarine + +``` +cd submarine-project-dir/ +mvn clean package -DskipTests +``` + +### Copy submarine jar to mini-submarine container + +``` +docker cp submarine-all/target/submarine-all--hadoop-.jar :/tmp/ +``` + +### Modify environment variables + +``` +cd /home/yarn/submarine +vi run_customized_submarine-all_mnist.sh + +# Need to modify environment variables based on hadoop and submarine version numbers +SUBMARINE_VERSION= +HADOOP_VERSION= # default 2.9 +``` + +### Test submarine jar package in container + +``` +cd /home/yarn/submarine +./run_customized_submarine-all_mnist.sh +``` + +## Debug Submarine + +When using mini-submarine, you can debug submarine client, applicationMaster and executor for trouble shooting. + +### Debug submarine client + +Run the following command to start mini-submarine. + +``` +docker run -it -P -h submarine-dev --net=bridge --expose=8000 --privileged local/mini-submarine: /bin/bash +``` + +Debug submarine client with the parameter "--debug" + +``` +./run_submarine_mnist_tony.sh --debug +``` + +Port 8000 is used in the mini-submarine. +You need to find the debug port mapping between mini-submarine and the host on which run mini-subamrine. + +``` +docker port +``` + +For example, we can get some info like this + +``` +8000/tcp -> 0.0.0.0:32804 +``` + +Then port 32804 can be used for remote debug. + +### Debug submarine job applicationMaster + +Run the following command to start mini-submarine. + +``` +docker run -it -P -h submarine-dev --net=bridge --expose=8001 --privileged local/mini-submarine: /bin/bash +``` + +Add the following configuration in the file /usr/local/hadoop/etc/hadoop/tony.xml. + +``` + + tony.task.am.jvm.opts + -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8001 + +``` + +You can use run_submarine_mnist_tony.sh to submit a job. Port 8001 is used for AM debugging in mini-submarine. +And the debug port mapping can be obtained using the way as [Debug submarine client](#debug) shows. + +### Debug submarine job executor + +Run the following command to start mini-submarine. + +``` +docker run -it -P -h submarine-dev --net=bridge --expose=8002 --privileged local/mini-submarine: /bin/bash +``` + +Add the following configuration in the file /usr/local/hadoop/etc/hadoop/tony.xml. + +``` + + tony.task.executor.jvm.opts + -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8002 + +``` + +Port 8002 is used for executor debugging in mini-submarine. +To avoid port confliction, you need to use only one executor, which means the parameter of +submarine job should be like this + +``` +--num_workers 1 \ +--num_ps 0 \ +``` + +You can get the debug port mapping using the way as [Debug submarine client](#debug) shows. + +## Run a distributedShell job with docker container + +You can also run a distributedShell job in mini-submarine. + +``` +cd && ./yarn-ds-docker.sh +``` + +## Run a spark job + +Spark jobs are supported as well. + +``` +cd && cd spark-script && ./run_spark.sh +``` + +## Question and answer + +1. Submarine package name error + + Because the package name of submarine 0.3.0 or higher has been changed from `apache.hadoop.yarn.submarine` to `apache.submarine`, So you need to set the Runtime settings in the `/usr/local/hadoop/etc/hadoop/submarine-site.xml` file. + + ``` + +    +      submarine.runtime.class + org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory +    + + ``` diff --git a/dev-support/mini-submarine/build_mini-submarine.sh b/dev-support/mini-submarine/build_mini-submarine.sh new file mode 100755 index 0000000000000000000000000000000000000000..43bd8ab7123e4ba5f9d58dea9c6f3cad8039d732 --- /dev/null +++ b/dev-support/mini-submarine/build_mini-submarine.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -euo pipefail +hadoop_v=2.9.2 +spark_v=2.3.4 + +submarine_v=${submarine_version:-"0.5.0"} +echo "Using submarine version: $submarine_v" + +image_name="local/mini-submarine:${submarine_v}" + +if [ -L ${BASH_SOURCE-$0} ]; then + PWD=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + PWD=$(dirname ${BASH_SOURCE-$0}) +fi +export MINI_PATH=$(cd "${PWD}">/dev/null; pwd) +SUBMARINE_PROJECT_PATH=${MINI_PATH}/../.. + +download_package() { + if [ -f "$1" ]; then + echo "Found $1" + else + echo "Start downloading the package $1 from $2" + if type wget >/dev/null 2>&1; then + wget $2/$1 + elif type curl >/dev/null 2>&1; then + curl -O $2/$1 + else + echo 'We need a tool to transfer data from or to a server. Such as wget/curl.' + echo 'Bye, bye!' + exit -1 + fi + fi +} + +is_empty_dir(){ + return `ls -A $1|wc -w` +} + +# download hadoop +download_package "hadoop-${hadoop_v}.tar.gz" "http://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-${hadoop_v}" +# download spark +download_package "spark-${spark_v}-bin-hadoop2.7.tgz" "http://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-${spark_v}" +# download zookeeper +download_package "zookeeper-3.4.14.tar.gz" "http://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.4.14" + + +if [ ! -d "${SUBMARINE_PROJECT_PATH}/submarine-dist/target" ]; then + mkdir "${SUBMARINE_PROJECT_PATH}/submarine-dist/target" +fi + +submarine_dist_exists=$(find -L "${SUBMARINE_PROJECT_PATH}/submarine-dist/target" -name "submarine-dist-${submarine_v}*.tar.gz") + +# If exists, use the release candidate artifacts to build image +if [[ ! -z "${release_candidates_path:-}" ]]; then + submarine_dist_exists=${release_candidates_path} + echo "Using release candidates artifacts: ${release_candidates_path}" + cp ${release_candidates_path}/submarine-dist-${submarine_v}-hadoop*.tar.gz ${MINI_PATH} +fi + +# Build source code if the package doesn't exist. +if [[ -z "${submarine_dist_exists}" ]]; then + cd "${SUBMARINE_PROJECT_PATH}" + mvn clean package -DskipTests +fi + +cp ${SUBMARINE_PROJECT_PATH}/submarine-dist/target/submarine-dist-${submarine_v}*.tar.gz ${MINI_PATH} +cp -r ${SUBMARINE_PROJECT_PATH}/submarine-sdk/pysubmarine ${MINI_PATH} +cp -r ${SUBMARINE_PROJECT_PATH}/docs/database ${MINI_PATH} + +# build image +echo "Start building the mini-submarine docker image..." +cd ${MINI_PATH} +docker build --build-arg HADOOP_VERSION=${hadoop_v} --build-arg SPARK_VERSION=${spark_v} --build-arg SUBMARINE_VERSION=${submarine_v} --build-arg IMAGE_NAME=${image_name} -t ${image_name} . + +# clean template file +rm -rf ${MINI_PATH}/database +rm -rf ${MINI_PATH}/pysubmarine +rm -rf ${MINI_PATH}/submarine-dist-${submarine_v}*.tar.gz diff --git a/dev-support/mini-submarine/conf/bootstrap.sh b/dev-support/mini-submarine/conf/bootstrap.sh new file mode 100755 index 0000000000000000000000000000000000000000..3152c6af32deada2500ed330b81a9c34ab856f87 --- /dev/null +++ b/dev-support/mini-submarine/conf/bootstrap.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# start dockerd +nohup dockerd --host=unix:///var/run/docker.sock > /var/log/dockerd.log 2>&1 & + +: ${HADOOP_PREFIX:=/usr/local/hadoop} + +. $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh + +# Directory to find config artifacts +CONFIG_DIR="/tmp/hadoop-config" + + +# Copy config files from volume mount + +for f in slaves core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml container-executor.cfg capacity-scheduler.xml node-resources.xml resource-types.xml; do + if [[ -e ${CONFIG_DIR}/$f ]]; then + cp ${CONFIG_DIR}/$f $HADOOP_PREFIX/etc/hadoop/$f + else + echo "ERROR: Could not find $f in $CONFIG_DIR" + exit 1 + fi +done + +# Copy submarine config +for f in slaves submarine-site.xml submarine-env.sh; do + if [[ -e ${CONFIG_DIR}/$f ]]; then + cp ${CONFIG_DIR}/$f /opt/submarine-current/conf/$f + else + echo "ERROR: Could not find $f in $CONFIG_DIR" + exit 1 + fi +done + +# create cgroups +mkdir -p /sys/fs/cgroup/cpu/hadoop-yarn +chown -R yarn /sys/fs/cgroup/cpu/hadoop-yarn +mkdir -p /sys/fs/cgroup/memory/hadoop-yarn +chown -R yarn /sys/fs/cgroup/memory/hadoop-yarn +mkdir -p /sys/fs/cgroup/blkio/hadoop-yarn +chown -R yarn /sys/fs/cgroup/blkio/hadoop-yarn +mkdir -p /sys/fs/cgroup/net_cls/hadoop-yarn +chown -R yarn /sys/fs/cgroup/net_cls/hadoop-yarn +mkdir -p /sys/fs/cgroup/devices/hadoop-yarn +chown -R yarn /sys/fs/cgroup/devices/hadoop-yarn + +# set container-executor permission +chmod 6050 /usr/local/hadoop/bin/container-executor +chmod 0400 /usr/local/hadoop/etc/hadoop/container-executor.cfg + +# creat log and app dir +mkdir ${HADOOP_PREFIX}/logs +chown yarn:hadoop ${HADOOP_PREFIX}/logs +mkdir /var/lib/hadoop-yarn +chown yarn:hadoop /var/lib/hadoop-yarn +mkdir /var/log/hadoop-yarn +chown yarn:hadoop /var/log/hadoop-yarn +# installing libraries if any - (resource urls added comma separated to the ACP system variable) +cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd - + +if [[ "${HOSTNAME}" =~ "submarine-dev" ]]; then + mkdir -p /root/hdfs/namenode + $HADOOP_PREFIX/bin/hdfs namenode -format -force -nonInteractive + sed -i s/hdfs-nn/0.0.0.0/ /usr/local/hadoop/etc/hadoop/core-site.xml + nohup $HADOOP_PREFIX/sbin/hadoop-daemon.sh start namenode > /tmp/nn.log 2>&1 & +fi + +if [[ "${HOSTNAME}" =~ "submarine-dev" ]]; then + mkdir -p /root/hdfs/datanode + # wait up to 30 seconds for namenode + # count=0 && while [[ $count -lt 15 && -z `curl -sf http://submarine-dev:50070` ]]; do echo "Waiting for hdfs-nn" ; ((count=count+1)) ; sleep 2; done + #[[ $count -eq 15 ]] && echo "Timeout waiting for hdfs-nn, exiting." && exit 1 + sleep 5 + nohup $HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode > /tmp/dn.log 2>&1 & +fi + +#add yarn to hdfs user group, create hdfs folder and export hadoop path +groupadd supergroup +usermod -aG supergroup yarn +usermod -aG docker yarn +su yarn -c "/usr/local/hadoop/bin/hadoop fs -mkdir -p /user/yarn" +echo "export PATH=$PATH:/usr/local/hadoop/bin" >> /home/yarn/.bashrc + +if [[ "${HOSTNAME}" =~ "submarine-dev" ]]; then + sed -i s/yarn-rm/0.0.0.0/ $HADOOP_PREFIX/etc/hadoop/yarn-site.xml + + # start zk + su yarn -c "cd /usr/local/zookeeper && /usr/local/zookeeper/bin/zkServer.sh start >/dev/null 2>&1" + + cp ${CONFIG_DIR}/start-yarn-rm.sh $HADOOP_PREFIX/sbin/ + cd $HADOOP_PREFIX/sbin + chmod +x start-yarn-rm.sh + su yarn -c "/usr/local/hadoop/sbin/start-yarn-rm.sh" +fi + +if [[ "${HOSTNAME}" =~ "submarine-dev" ]]; then + sed -i '/<\/configuration>/d' $HADOOP_PREFIX/etc/hadoop/yarn-site.xml + cat >> $HADOOP_PREFIX/etc/hadoop/yarn-site.xml <<- EOM + + yarn.nodemanager.resource.memory-mb + 8192 + + + + yarn.nodemanager.resource.cpu-vcores + 16 + +EOM + echo '' >> $HADOOP_PREFIX/etc/hadoop/yarn-site.xml + cp ${CONFIG_DIR}/start-yarn-nm.sh $HADOOP_PREFIX/sbin/ + cd $HADOOP_PREFIX/sbin + chmod +x start-yarn-nm.sh + + #copy the distributed shell script for debug purpose + cp ${CONFIG_DIR}/yarn-ds-docker.sh /home/yarn/ + chown yarn /home/yarn/yarn-ds-docker.sh + chmod +x /home/yarn/yarn-ds-docker.sh + + # wait up to 30 seconds for resourcemanager + count=0 && while [[ $count -lt 30 && -z `curl -sf http://submarine-dev:8088/ws/v1/cluster/info` ]]; do echo "Waiting for yarn-rm" ; ((count=count+1)) ; sleep 1; done + [[ $count -eq 30 ]] && echo "Timeout waiting for yarn-rm, exiting." && exit 1 + + su yarn -c "/usr/local/hadoop/sbin/start-yarn-nm.sh" +fi + +if [[ $1 == "-d" ]]; then + until find ${HADOOP_PREFIX}/logs -mmin -1 | egrep -q '.*'; echo "`date`: Waiting for logs..." ; do sleep 2 ; done + tail -F ${HADOOP_PREFIX}/logs/* & + while true; do sleep 1000; done +fi + +if [[ $1 == "-bash" ]]; then + /bin/bash +fi diff --git a/dev-support/mini-submarine/conf/capacity-scheduler.xml b/dev-support/mini-submarine/conf/capacity-scheduler.xml new file mode 100644 index 0000000000000000000000000000000000000000..cc5ccb128f73af4a29b8b4a5c10550480e75da99 --- /dev/null +++ b/dev-support/mini-submarine/conf/capacity-scheduler.xml @@ -0,0 +1,73 @@ + + + + + + + yarn.scheduler.capacity.root.queues + default + + + yarn.scheduler.capacity.root.default.capacity + 100 + + + yarn.scheduler.capacity.maximum-am-resource-percent + 1 + + + yarn.scheduler.capacity.per-node-heartbeat.maximum-container-assignments + 100 + + + + yarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments + 100 + + + + + yarn.scheduler.capacity.resource-calculator + org.apache.hadoop.yarn.util.resource.DominantResourceCalculator + + + + yarn.scheduler.capacity.multi-node-placement-enabled + true + + + + yarn.scheduler.capacity.multi-node-sorting.policy.names + resource-based + + + + yarn.scheduler.capacity.multi-node-sorting.policy + resource-based + + + + yarn.scheduler.capacity.multi-node-sorting.policy.resource-based.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceUsageMultiNodeLookupPolicy + + diff --git a/dev-support/mini-submarine/conf/container-executor.cfg b/dev-support/mini-submarine/conf/container-executor.cfg new file mode 100644 index 0000000000000000000000000000000000000000..4feaeca493eca5a4fbe195e69836385a42d02613 --- /dev/null +++ b/dev-support/mini-submarine/conf/container-executor.cfg @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +yarn.nodemanager.linux-container-executor.group=hadoop +banned.users=test +min.user.id=1000 +#allowed.system.users=##comma separated list of system users who CAN run applications +feature.tc.enabled=false + +# The configs below deal with settings for Docker +[docker] + module.enabled=true + docker.binary=/usr/bin/docker +# docker.allowed.devices=## comma separated list of devices that can be mounted into a container + docker.privileged-containers.enabled=false +# docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers +# docker.no-new-privileges.enabled=## enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default + docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE + docker.allowed.networks=bridge,host,none + docker.trusted.registries=tangzhankun,library + docker.allowed.ro-mounts=/sys/fs/cgroup,/etc/passwd,/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/filecache,/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/yarn/filecache,/etc/passwd + docker.allowed.rw-mounts=/var/log/hadoop-yarn/containers,/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/usercache/yarn,/var/lib/hadoop-yarn/cache/yarn/nm-local-dir/filecache + docker.allowed.runtimes=runc,nvidia + docker.allowed.devices=/dev/nvidiactl,/dev/nvidia-uvm,/dev/nvidia-uvm-tools,/dev/nvidia0,/dev/nvidia-modeset +# docker.allowed.rw-mounts=/opt/code/hadoop/hadoop-dist/target/hadoop-3.3.0-SNAPSHOT/logs,/tmp/hadoop-yarn/nm-local-dir,/home/yarn/apache-hadoop-install-dir/logs/yarn/userlogs + +# The configs below deal with settings for FPGA resource +#[fpga] +# module.enabled=## Enable/Disable the FPGA resource handler module. set to "true" to enable, disabled by default +# fpga.major-device-number=## Major device number of FPGA, by default is 246. Strongly recommend setting this +# fpga.allowed-device-minor-numbers=## Comma separated allowed minor device numbers, empty means all FPGA devices managed by +[cgroups] + root=/sys/fs/cgroup + yarn-hierarchy=hadoop-yarn +#[gpu] +# module.enabled=true +#[devices] +# module.enabled=true +# devices.allowed-numbers=8:32,8:48 diff --git a/dev-support/mini-submarine/conf/core-site.xml b/dev-support/mini-submarine/conf/core-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..1af90a714f151cae28fb87ed8f7b4cc154ec3dc7 --- /dev/null +++ b/dev-support/mini-submarine/conf/core-site.xml @@ -0,0 +1,25 @@ + + + + + + fs.defaultFS + hdfs://hdfs-nn:9000/ + NameNode URI + + diff --git a/dev-support/mini-submarine/conf/exclude-nodes.txt b/dev-support/mini-submarine/conf/exclude-nodes.txt new file mode 100644 index 0000000000000000000000000000000000000000..09697dce6e11859e38a9bfb321bfc72eb88b8ce6 --- /dev/null +++ b/dev-support/mini-submarine/conf/exclude-nodes.txt @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/dev-support/mini-submarine/conf/excludeNodes.txt b/dev-support/mini-submarine/conf/excludeNodes.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/dev-support/mini-submarine/conf/hdfs-site.xml b/dev-support/mini-submarine/conf/hdfs-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..a1d46b2d410496baa99a75e5dd902802a4021182 --- /dev/null +++ b/dev-support/mini-submarine/conf/hdfs-site.xml @@ -0,0 +1,52 @@ + + + + + + + dfs.datanode.use.datanode.hostname + false + + + + dfs.client.use.datanode.hostname + false + + + + dfs.replication + 3 + + + + dfs.datanode.data.dir + file:///root/hdfs/datanode + DataNode directory + + + + dfs.namenode.name.dir + file:///root/hdfs/namenode + NameNode directory for namespace and transaction logs storage. + + + + dfs.namenode.datanode.registration.ip-hostname-check + false + + diff --git a/dev-support/mini-submarine/conf/include-nodes.txt b/dev-support/mini-submarine/conf/include-nodes.txt new file mode 100644 index 0000000000000000000000000000000000000000..09697dce6e11859e38a9bfb321bfc72eb88b8ce6 --- /dev/null +++ b/dev-support/mini-submarine/conf/include-nodes.txt @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/dev-support/mini-submarine/conf/mapred-site.xml b/dev-support/mini-submarine/conf/mapred-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..22b86b9ea340ebf67672f5eeea84c3c360ea462e --- /dev/null +++ b/dev-support/mini-submarine/conf/mapred-site.xml @@ -0,0 +1,33 @@ + + + + + + + mapreduce.framework.name + yarn + + + mapreduce.jobhistory.address + yarn-rm-0.yarn-rm.yarn-cluster.svc.cluster.local:10020 + + + mapreduce.jobhistory.webapp.address + yarn-rm-0.yarn-rm.yarn-cluster.svc.cluster.local:19888 + + diff --git a/dev-support/mini-submarine/conf/node-resources.xml b/dev-support/mini-submarine/conf/node-resources.xml new file mode 100644 index 0000000000000000000000000000000000000000..743c6d86970ed989f130516b01b4138126bc2897 --- /dev/null +++ b/dev-support/mini-submarine/conf/node-resources.xml @@ -0,0 +1,25 @@ + + + + + diff --git a/dev-support/mini-submarine/conf/resource-types.xml b/dev-support/mini-submarine/conf/resource-types.xml new file mode 100644 index 0000000000000000000000000000000000000000..4bc43c4dceb9f062c2f33023b64dbfde9cf267ef --- /dev/null +++ b/dev-support/mini-submarine/conf/resource-types.xml @@ -0,0 +1,25 @@ + + + + + diff --git a/dev-support/mini-submarine/conf/setup-mysql.sh b/dev-support/mini-submarine/conf/setup-mysql.sh new file mode 100755 index 0000000000000000000000000000000000000000..d59632b2afaa0266d91c74206a61ad6bcb8d731e --- /dev/null +++ b/dev-support/mini-submarine/conf/setup-mysql.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install mariadb +apt-get -y install mariadb-server +service mysql start +mysql -e "CREATE DATABASE submarine_test;" +mysql -e "CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';" +mysql -e "use submarine_test; source /home/yarn/database/submarine.sql; source /home/yarn/database/submarine-data.sql;" + +mysql -e "CREATE DATABASE submarine;" +mysql -e "CREATE USER 'submarine'@'%' IDENTIFIED BY 'password';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';" +mysql -e "use submarine; source /home/yarn/database/submarine.sql; source /home/yarn/database/submarine-data.sql;" + +mysql -e "CREATE DATABASE metastore_test;" +mysql -e "CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'metastore_test'@'%';" +mysql -e "use metastore_test; source /home/yarn/database/metastore.sql;" + +mysql -e "CREATE DATABASE metastore;" +mysql -e "CREATE USER 'metastore'@'%' IDENTIFIED BY 'password';" +mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%';" +mysql -e "use metastore; source /home/yarn/database/metastore.sql;" diff --git a/dev-support/mini-submarine/conf/slaves b/dev-support/mini-submarine/conf/slaves new file mode 100644 index 0000000000000000000000000000000000000000..63fb8bb82bbec04ec08ac7ce51d106afd797eecd --- /dev/null +++ b/dev-support/mini-submarine/conf/slaves @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +localhost diff --git a/dev-support/mini-submarine/conf/start-yarn-nm.sh b/dev-support/mini-submarine/conf/start-yarn-nm.sh new file mode 100755 index 0000000000000000000000000000000000000000..7fb683f90829cca027232be466813e04066750f0 --- /dev/null +++ b/dev-support/mini-submarine/conf/start-yarn-nm.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start all yarn daemons. Run this on master node. + +echo "starting yarn daemons" + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin"; pwd` + +DEFAULT_LIBEXEC_DIR="$bin"/../libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# start resourceManager +# "$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager +# start nodeManager +"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start nodemanager +# start proxyserver +#"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start proxyserver diff --git a/dev-support/mini-submarine/conf/start-yarn-rm.sh b/dev-support/mini-submarine/conf/start-yarn-rm.sh new file mode 100755 index 0000000000000000000000000000000000000000..419e70cf8d629df1820fe60821b9bf3265ab5f04 --- /dev/null +++ b/dev-support/mini-submarine/conf/start-yarn-rm.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start all yarn daemons. Run this on master node. + +echo "starting yarn daemons" + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin"; pwd` + +DEFAULT_LIBEXEC_DIR="$bin"/../libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# start resourceManager +"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager +# start nodeManager +# "$bin"/yarn-daemons.sh --config $YARN_CONF_DIR start nodemanager +# start proxyserver +"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start proxyserver diff --git a/dev-support/mini-submarine/conf/stop-yarn-nm.sh b/dev-support/mini-submarine/conf/stop-yarn-nm.sh new file mode 100755 index 0000000000000000000000000000000000000000..03b57d60421fe7854944aef8ac0bbdd8d0f5fff2 --- /dev/null +++ b/dev-support/mini-submarine/conf/stop-yarn-nm.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start all yarn daemons. Run this on master node. + +echo "starting yarn daemons" + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin"; pwd` + +DEFAULT_LIBEXEC_DIR="$bin"/../libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# Stop nodeManager +"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop nodemanager diff --git a/dev-support/mini-submarine/conf/stop-yarn-rm.sh b/dev-support/mini-submarine/conf/stop-yarn-rm.sh new file mode 100755 index 0000000000000000000000000000000000000000..b5b42a6a7a36830992814eae78fef954025ff41f --- /dev/null +++ b/dev-support/mini-submarine/conf/stop-yarn-rm.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start all yarn daemons. Run this on master node. + +echo "starting yarn daemons" + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin"; pwd` + +DEFAULT_LIBEXEC_DIR="$bin"/../libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# stop resourceManager +"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop resourcemanager +# stop proxyserver +"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop proxyserver diff --git a/dev-support/mini-submarine/conf/submarine-env.sh b/dev-support/mini-submarine/conf/submarine-env.sh new file mode 100644 index 0000000000000000000000000000000000000000..64ea5654e606f380376dc9b3749ecf73439f9091 --- /dev/null +++ b/dev-support/mini-submarine/conf/submarine-env.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# export JAVA_HOME=java + +# Debug Submarine server +# export SUBMARINE_SERVER_JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000" + +# Set Submarine server memory +# export SUBMARINE_SERVER_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m" + +# Set Submarine server classpath. If you want to visit hdfs, just add hadoop +# configuration path. +export SUBMARINE_SERVER_CLASSPATH+=":/usr/local/hadoop/etc/hadoop" diff --git a/dev-support/mini-submarine/conf/submarine-site.xml b/dev-support/mini-submarine/conf/submarine-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..f2236ae5a97818a20082fb946bf5dd574818de15 --- /dev/null +++ b/dev-support/mini-submarine/conf/submarine-site.xml @@ -0,0 +1,122 @@ + + + + + + + + submarine.cluster.addr + + submarine cluster address list, e.g. ip1:port1;ip2:port2;ip3:port3 + + + + submarine.server.addr + 0.0.0.0 + Server address + + + + submarine.server.port + 8080 + Server port. + + + + submarine.server.ssl + false + Should SSL be used by the submarine server? + + + + submarine.server.ssl.port + 8443 + Server ssl port. (used when ssl property is set to true) + + + + submarine.server.ssl.client.auth + false + Should client authentication be used for SSL connections? + + + + submarine.server.ssl.keystore.path + keystore + Path to keystore relative to submarine configuration directory + + + + submarine.server.ssl.keystore.type + JKS + The format of the given keystore (e.g. JKS or PKCS12) + + + + submarine.server.ssl.keystore.password + change me + Keystore password. Can be obfuscated by the Jetty Password tool + + + + + + submarine.server.ssl.truststore.path + truststore + Path to truststore relative to submarine configuration directory. Defaults to the keystore path + + + + submarine.server.ssl.truststore.type + JKS + The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type + + + + + + workbench.web.war + ../submarine-workbench-web.war + Submarine workbench web war file path. + + + + submarine.runtime.class + org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory + RuntimeFactory for Submarine jobs + + + + submarine.server.rpc.enabled + false + Run jobs using rpc server. + + + diff --git a/dev-support/mini-submarine/conf/tony.xml b/dev-support/mini-submarine/conf/tony.xml new file mode 100644 index 0000000000000000000000000000000000000000..f8c3ef86bd99c97d660872bab762f24e386b44e3 --- /dev/null +++ b/dev-support/mini-submarine/conf/tony.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dev-support/mini-submarine/conf/yarn-ds-docker.sh b/dev-support/mini-submarine/conf/yarn-ds-docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..5232e03f11a74a20ff333e3c9aa551c4ed392a6e --- /dev/null +++ b/dev-support/mini-submarine/conf/yarn-ds-docker.sh @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +DS=/usr/local/hadoop/share/hadoop/yarn/hadoop-yarn-applications-distributedshell-${HADOOP_VER}.jar +/usr/local/hadoop/bin/yarn jar ${DS} -jar ${DS} -shell_command "cat /proc/1/cgroup" -shell_env YARN_CONTAINER_RUNTIME_TYPE=docker -shell_env YARN_CONTAINER_RUNTIME_DOCKER_IMAGE=library/ubuntu diff --git a/dev-support/mini-submarine/conf/yarn-site.xml b/dev-support/mini-submarine/conf/yarn-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..1da453d39b5ffa3441315503a7f25cf3a31d5ff2 --- /dev/null +++ b/dev-support/mini-submarine/conf/yarn-site.xml @@ -0,0 +1,185 @@ + + + + + + + yarn.resourcemanager.hostname + yarn-rm + + + + yarn.resourcemanager.scheduler.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + + yarn.nodemanager.vmem-check-enabled + false + + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + + List of directories to store localized files in. + yarn.nodemanager.local-dirs + /var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir + + + + Where to store container logs. + yarn.nodemanager.log-dirs + /var/log/hadoop-yarn/containers + + + + Where to aggregate logs to. + yarn.nodemanager.remote-app-log-dir + /var/log/hadoop-yarn/apps + + + + yarn.application.classpath + + /usr/local/hadoop/etc/hadoop, + /usr/local/hadoop/share/hadoop/common/*, + /usr/local/hadoop/share/hadoop/common/lib/*, + /usr/local/hadoop/share/hadoop/hdfs/*, + /usr/local/hadoop/share/hadoop/hdfs/lib/*, + /usr/local/hadoop/share/hadoop/mapreduce/*, + /usr/local/hadoop/share/hadoop/mapreduce/lib/*, + /usr/local/hadoop/share/hadoop/yarn/*, + /usr/local/hadoop/share/hadoop/yarn/lib/* + + + + yarn.log-aggregation-enable + true + + + + yarn.log-aggregation.retain-seconds + 259200 + + + yarn.log-aggregation.retain-check-interval-seconds + 3600 + + + yarn.nodemanager.delete.debug-delay-sec + 86400 + + + yarn.nodemanager.container-executor.class + org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor + + + + + yarn.nodemanager.linux-container-executor.group + hadoop + + + yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user + yarn + + + + yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users + false + + + + yarn.nodemanager.runtime.linux.allowed-runtimes + default,docker + + + + yarn.nodemanager.runtime.linux.docker.allowed-container-networks + host,none,bridge + + + yarn.nodemanager.runtime.linux.docker.default-container-network + host + + + + yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed + false + + + yarn.nodemanager.runtime.linux.docker.privileged-containers.acl + + + + + yarn.nodemanager.runtime.linux.docker.capabilities + CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE + + + + yarn.nodemanager.linux-container-executor.resources-handler.class + org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler + + + yarn.nodemanager.linux-container-executor.cgroups.mount + false + + + yarn.nodemanager.linux-container-executor.cgroups.hierarchy + /hadoop-yarn + + + yarn.nodemanager.linux-container-executor.cgroups.mount-path + /sys/fs/cgroup + + + yarn.nodemanager.resource.percentage-physical-cpu-limit + 100 + + + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage + false + + + + + + yarn.webapp.api-service.enable + true + + + diff --git a/dev-support/mini-submarine/spark-defaults-dynamic-allocation.conf b/dev-support/mini-submarine/spark-defaults-dynamic-allocation.conf new file mode 100644 index 0000000000000000000000000000000000000000..1cc360ea4cb5b92605b9188db02dd8d89b8aca7e --- /dev/null +++ b/dev-support/mini-submarine/spark-defaults-dynamic-allocation.conf @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default system properties included when running spark-submit. +# This is useful for setting default environmental settings. + +#spark.yarn.stagingDir /tmp/staging +spark.yarn.preserve.staging.files true +#spark.dynamicAllocation.enabled true +spark.shuffle.service.enabled true +spark.dynamicAllocation.minExecutors 1 + +# Example: +# spark.master spark://master:7077 +# spark.eventLog.enabled true +# spark.eventLog.dir hdfs://namenode:8021/directory +# spark.serializer org.apache.spark.serializer.KryoSerializer +# spark.driver.memory 5g +# spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three" diff --git a/dev-support/mini-submarine/spark-script/pyspark-yarn.py b/dev-support/mini-submarine/spark-script/pyspark-yarn.py new file mode 100755 index 0000000000000000000000000000000000000000..a28a5a206fc3f3002ffb1b9229123a159dd05b9b --- /dev/null +++ b/dev-support/mini-submarine/spark-script/pyspark-yarn.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pyspark import SparkConf +from pyspark import SparkContext + +conf = SparkConf() +conf.setMaster('yarn-client') +conf.setAppName('spark-yarn') +sc = SparkContext(conf=conf) + + +def mod(x): + #import time + #time.sleep(120) + return x + +rdd = sc.parallelize(range(1000)).map(mod).take(10) +print rdd diff --git a/dev-support/mini-submarine/spark-script/run_gpu_ds.sh b/dev-support/mini-submarine/spark-script/run_gpu_ds.sh new file mode 100755 index 0000000000000000000000000000000000000000..26321a43d3697ffd5c91ed459e9c8bae2b18c347 --- /dev/null +++ b/dev-support/mini-submarine/spark-script/run_gpu_ds.sh @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash + +#JAR_PATH=/usr/local/hadoop/share/hadoop/yarn/hadoop-yarn-applications-distributedshell-${HADOOP_VER}.jar + +#yarn jar $JAR_PATH -jar $JAR_PATH -shell_command "sleep 5" -master_resources memory-mb=512,vcores=2,nvidia.com/gpu=1 -container_resources memory-mb=512,vcores=1 diff --git a/dev-support/mini-submarine/spark-script/run_pyspark.sh b/dev-support/mini-submarine/spark-script/run_pyspark.sh new file mode 100755 index 0000000000000000000000000000000000000000..d834c87ef6f406c3a04f867e1b19159871394176 --- /dev/null +++ b/dev-support/mini-submarine/spark-script/run_pyspark.sh @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +SPARK_HOME=/opt/spark-${SPARK_VER}-bin-hadoop2.7 +${SPARK_HOME}/bin/spark-submit \ +--master yarn \ +--deploy-mode client \ +--driver-memory 1g \ +--executor-memory 2g \ +--num-executors 1 \ +--executor-cores 4 \ +./pyspark-yarn.py diff --git a/dev-support/mini-submarine/spark-script/run_pyspark_docker.sh b/dev-support/mini-submarine/spark-script/run_pyspark_docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..f4156c33dcca244b87c3d258208dd71d6ca3daf4 --- /dev/null +++ b/dev-support/mini-submarine/spark-script/run_pyspark_docker.sh @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +SPARK_HOME=/opt/spark-${SPARK_VER}-bin-hadoop2.7 +${SPARK_HOME}/bin/spark-submit \ +--master yarn \ +--deploy-mode client \ +--driver-memory 1g \ +--executor-memory 1g \ +--executor-cores 1 \ +--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_TYPE=docker \ +--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_IMAGE= \ +--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS=/etc/passwd:/etc/passwd:ro \ +./pyspark-yarn.py diff --git a/dev-support/mini-submarine/spark-script/run_spark.sh b/dev-support/mini-submarine/spark-script/run_spark.sh new file mode 100755 index 0000000000000000000000000000000000000000..f44b12cc69964f6f981948697aa0c7364598c120 --- /dev/null +++ b/dev-support/mini-submarine/spark-script/run_spark.sh @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash + +SPARK_HOME=/opt/spark-${SPARK_VER}-bin-hadoop2.7 +${SPARK_HOME}/bin/spark-submit --class org.apache.spark.examples.SparkPi \ +--master yarn \ +--deploy-mode client \ +--driver-memory 512m \ +--executor-memory 512m \ +--executor-cores 1 \ +${SPARK_HOME}/examples/jars/spark-examples*.jar \ +10 diff --git a/dev-support/mini-submarine/spark-script/run_spark_docker.sh b/dev-support/mini-submarine/spark-script/run_spark_docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..8b1078f1216d36631c439978de20ff1124df942a --- /dev/null +++ b/dev-support/mini-submarine/spark-script/run_spark_docker.sh @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +SPARK_HOME=/opt/spark-${SPARK_VER}-bin-hadoop2.7 +${SPARK_HOME}/bin/spark-submit --class org.apache.spark.examples.SparkPi \ +--master yarn \ +--deploy-mode client \ +--driver-memory 512m \ +--executor-memory 512m \ +--executor-cores 1 \ +--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_TYPE=docker \ +--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_IMAGE= \ +--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS=/etc/passwd:/etc/passwd:ro \ +--conf spark.dynamicAllocation.enabled=false \ +${SPARK_HOME}/examples/jars/spark-examples*.jar \ +10 diff --git a/dev-support/mini-submarine/submarine/build_python_virtual_env.sh b/dev-support/mini-submarine/submarine/build_python_virtual_env.sh new file mode 100755 index 0000000000000000000000000000000000000000..fcb9d14e18b08ef4b718a1b5c74c30edf96185fb --- /dev/null +++ b/dev-support/mini-submarine/submarine/build_python_virtual_env.sh @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +wget https://files.pythonhosted.org/packages/33/bc/fa0b5347139cd9564f0d44ebd2b147ac97c36b2403943dbee8a25fd74012/virtualenv-16.0.0.tar.gz +tar xf virtualenv-16.0.0.tar.gz + +# Make sure to install using Python 3, as TensorFlow only provides Python 3 artifacts +python3 virtualenv-16.0.0/virtualenv.py venv +. venv/bin/activate +pip3 install tensorflow==1.13.1 +pip3 install torch==0.4.1 +pip3 install torchvision==0.1.8 +pip3 install mxnet==1.5.1 +pip3 install /opt/pysubmarine/. +zip -r myvenv.zip venv +deactivate + +# Building a Python virtual environment with TensorFlow 2 +python3 virtualenv-16.0.0/virtualenv.py tf2-venv +. tf2-venv/bin/activate +pip3 install tensorflow==2.1.0 +pip3 install tensorflow-datasets==2.1.0 +zip -r tf2-venv.zip tf2-venv +deactivate diff --git a/dev-support/mini-submarine/submarine/image_classification.py b/dev-support/mini-submarine/submarine/image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..14eb68cd7935274ad6f95f30037e4bc4b5e64ce0 --- /dev/null +++ b/dev-support/mini-submarine/submarine/image_classification.py @@ -0,0 +1,465 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division + +import argparse +import time +import os +import logging +import random +import tarfile + +import mxnet as mx +from mxnet import gluon +from mxnet import profiler +from mxnet.gluon import nn +from mxnet.gluon.model_zoo import vision as models +from mxnet.gluon.data.vision import ImageFolderDataset +from mxnet.gluon.data import DataLoader +from mxnet.contrib.io import DataLoaderIter +from mxnet import autograd as ag +from mxnet.test_utils import get_mnist_iterator, get_cifar10 +from mxnet.metric import Accuracy, TopKAccuracy, CompositeEvalMetric +import numpy as np + +# logging +logging.basicConfig(level=logging.INFO) +fh = logging.FileHandler('image-classification.log') +logger = logging.getLogger() +logger.addHandler(fh) +formatter = logging.Formatter('%(message)s') +fh.setFormatter(formatter) +fh.setLevel(logging.DEBUG) +logging.debug('\n%s', '-' * 100) +formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') +fh.setFormatter(formatter) + +# CLI +parser = argparse.ArgumentParser(description='Train a model for image classification.') +parser.add_argument('--dataset', type=str, default='cifar10', + help='dataset to use. options are mnist, cifar10, caltech101, imagenet and dummy.') +parser.add_argument('--data-dir', type=str, default='', + help='training directory of imagenet images, contains train/val subdirs.') +parser.add_argument('--num-worker', '-j', dest='num_workers', default=4, type=int, + help='number of workers for dataloader') +parser.add_argument('--batch-size', type=int, default=32, + help='training batch size per device (CPU/GPU).') +parser.add_argument('--gpus', type=str, default='', + help='ordinates of gpus to use, can be "0,1,2" or empty for cpu only.') +parser.add_argument('--epochs', type=int, default=120, + help='number of training epochs.') +parser.add_argument('--lr', type=float, default=0.1, + help='learning rate. default is 0.1.') +parser.add_argument('--momentum', type=float, default=0.9, + help='momentum value for optimizer, default is 0.9.') +parser.add_argument('--wd', type=float, default=0.0001, + help='weight decay rate. default is 0.0001.') +parser.add_argument('--seed', type=int, default=123, + help='random seed to use. Default=123.') +parser.add_argument('--mode', type=str, + help='mode in which to train the model. options are symbolic, imperative, hybrid') +parser.add_argument('--model', type=str, required=True, + help='type of model to use. see vision_model for options.') +parser.add_argument('--use_thumbnail', action='store_true', + help='use thumbnail or not in resnet. default is false.') +parser.add_argument('--batch-norm', action='store_true', + help='enable batch normalization or not in vgg. default is false.') +parser.add_argument('--use-pretrained', action='store_true', + help='enable using pretrained model from gluon.') +parser.add_argument('--prefix', default='', type=str, + help='path to checkpoint prefix, default is current working dir') +parser.add_argument('--start-epoch', default=0, type=int, + help='starting epoch, 0 for fresh training, > 0 to resume') +parser.add_argument('--resume', type=str, default='', + help='path to saved weight where you want resume') +parser.add_argument('--lr-factor', default=0.1, type=float, + help='learning rate decay ratio') +parser.add_argument('--lr-steps', default='30,60,90', type=str, + help='list of learning rate decay epochs as in str') +parser.add_argument('--dtype', default='float32', type=str, + help='data type, float32 or float16 if applicable') +parser.add_argument('--save-frequency', default=10, type=int, + help='epoch frequence to save model, best model will always be saved') +parser.add_argument('--kvstore', type=str, default='device', + help='kvstore to use for trainer/module.') +parser.add_argument('--log-interval', type=int, default=50, + help='Number of batches to wait before logging.') +parser.add_argument('--profile', action='store_true', + help='Option to turn on memory profiling for front-end, '\ + 'and prints out the memory usage by python function at the end.') +parser.add_argument('--builtin-profiler', type=int, default=0, help='Enable built-in profiler (0=off, 1=on)') +opt = parser.parse_args() + +# global variables +logger.info('Starting new image-classification task:, %s',opt) +mx.random.seed(opt.seed) +model_name = opt.model +dataset_classes = {'mnist': 10, 'cifar10': 10, 'caltech101':101, 'imagenet': 1000, 'dummy': 1000} +batch_size, dataset, classes = opt.batch_size, opt.dataset, dataset_classes[opt.dataset] +context = [mx.gpu(int(i)) for i in opt.gpus.split(',')] if opt.gpus.strip() else [mx.cpu()] +num_gpus = len(context) +batch_size *= max(1, num_gpus) +lr_steps = [int(x) for x in opt.lr_steps.split(',') if x.strip()] +metric = CompositeEvalMetric([Accuracy(), TopKAccuracy(5)]) +kv = mx.kv.create(opt.kvstore) + + +def get_cifar10_iterator(batch_size, data_shape, resize=-1, num_parts=1, part_index=0): + get_cifar10() + + train = mx.io.ImageRecordIter( + path_imgrec = "data/cifar/train.rec", + resize = resize, + data_shape = data_shape, + batch_size = batch_size, + rand_crop = True, + rand_mirror = True, + num_parts=num_parts, + part_index=part_index) + + val = mx.io.ImageRecordIter( + path_imgrec = "data/cifar/test.rec", + resize = resize, + rand_crop = False, + rand_mirror = False, + data_shape = data_shape, + batch_size = batch_size, + num_parts=num_parts, + part_index=part_index) + + return train, val + +def get_imagenet_transforms(data_shape=224, dtype='float32'): + def train_transform(image, label): + image, _ = mx.image.random_size_crop(image, (data_shape, data_shape), 0.08, (3/4., 4/3.)) + image = mx.nd.image.random_flip_left_right(image) + image = mx.nd.image.to_tensor(image) + image = mx.nd.image.normalize(image, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) + return mx.nd.cast(image, dtype), label + + def val_transform(image, label): + image = mx.image.resize_short(image, data_shape + 32) + image, _ = mx.image.center_crop(image, (data_shape, data_shape)) + image = mx.nd.image.to_tensor(image) + image = mx.nd.image.normalize(image, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) + return mx.nd.cast(image, dtype), label + return train_transform, val_transform + +def get_imagenet_iterator(root, batch_size, num_workers, data_shape=224, dtype='float32'): + """Dataset loader with preprocessing.""" + train_dir = os.path.join(root, 'train') + train_transform, val_transform = get_imagenet_transforms(data_shape, dtype) + logging.info("Loading image folder %s, this may take a bit long...", train_dir) + train_dataset = ImageFolderDataset(train_dir, transform=train_transform) + train_data = DataLoader(train_dataset, batch_size, shuffle=True, + last_batch='discard', num_workers=num_workers) + val_dir = os.path.join(root, 'val') + if not os.path.isdir(os.path.expanduser(os.path.join(root, 'val', 'n01440764'))): + user_warning = 'Make sure validation images are stored in one subdir per category, a helper script is available at https://git.io/vNQv1' + raise ValueError(user_warning) + logging.info("Loading image folder %s, this may take a bit long...", val_dir) + val_dataset = ImageFolderDataset(val_dir, transform=val_transform) + val_data = DataLoader(val_dataset, batch_size, last_batch='keep', num_workers=num_workers) + return DataLoaderIter(train_data, dtype), DataLoaderIter(val_data, dtype) + +def get_caltech101_data(): + url = "https://s3.us-east-2.amazonaws.com/mxnet-public/101_ObjectCategories.tar.gz" + dataset_name = "101_ObjectCategories" + data_folder = "data" + if not os.path.isdir(data_folder): + os.makedirs(data_folder) + tar_path = mx.gluon.utils.download(url, path=data_folder) + if (not os.path.isdir(os.path.join(data_folder, "101_ObjectCategories")) or + not os.path.isdir(os.path.join(data_folder, "101_ObjectCategories_test"))): + tar = tarfile.open(tar_path, "r:gz") + tar.extractall(data_folder) + tar.close() + print('Data extracted') + training_path = os.path.join(data_folder, dataset_name) + testing_path = os.path.join(data_folder, "{}_test".format(dataset_name)) + return training_path, testing_path + +def get_caltech101_iterator(batch_size, num_workers, dtype): + def transform(image, label): + # resize the shorter edge to 224, the longer edge will be greater or equal to 224 + resized = mx.image.resize_short(image, 224) + # center and crop an area of size (224,224) + cropped, crop_info = mx.image.center_crop(resized, (224, 224)) + # transpose the channels to be (3,224,224) + transposed = mx.nd.transpose(cropped, (2, 0, 1)) + return transposed, label + + training_path, testing_path = get_caltech101_data() + dataset_train = ImageFolderDataset(root=training_path, transform=transform) + dataset_test = ImageFolderDataset(root=testing_path, transform=transform) + + train_data = DataLoader(dataset_train, batch_size, shuffle=True, num_workers=num_workers) + test_data = DataLoader(dataset_test, batch_size, shuffle=False, num_workers=num_workers) + return DataLoaderIter(train_data), DataLoaderIter(test_data) + +class DummyIter(mx.io.DataIter): + def __init__(self, batch_size, data_shape, batches = 100): + super(DummyIter, self).__init__(batch_size) + self.data_shape = (batch_size,) + data_shape + self.label_shape = (batch_size,) + self.provide_data = [('data', self.data_shape)] + self.provide_label = [('softmax_label', self.label_shape)] + self.batch = mx.io.DataBatch(data=[mx.nd.zeros(self.data_shape)], + label=[mx.nd.zeros(self.label_shape)]) + self._batches = 0 + self.batches = batches + + def next(self): + if self._batches < self.batches: + self._batches += 1 + return self.batch + else: + self._batches = 0 + raise StopIteration + +def dummy_iterator(batch_size, data_shape): + return DummyIter(batch_size, data_shape), DummyIter(batch_size, data_shape) + +class ImagePairIter(mx.io.DataIter): + def __init__(self, path, data_shape, label_shape, batch_size=64, flag=0, input_aug=None, target_aug=None): + super(ImagePairIter, self).__init__(batch_size) + self.data_shape = (batch_size,) + data_shape + self.label_shape = (batch_size,) + label_shape + self.input_aug = input_aug + self.target_aug = target_aug + self.provide_data = [('data', self.data_shape)] + self.provide_label = [('label', self.label_shape)] + is_image_file = lambda fn: any(fn.endswith(ext) for ext in [".png", ".jpg", ".jpeg"]) + self.filenames = [os.path.join(path, x) for x in os.listdir(path) if is_image_file(x)] + self.count = 0 + self.flag = flag + random.shuffle(self.filenames) + + def next(self): + from PIL import Image + if self.count + self.batch_size <= len(self.filenames): + data = [] + label = [] + for i in range(self.batch_size): + fn = self.filenames[self.count] + self.count += 1 + image = Image.open(fn).convert('YCbCr').split()[0] + if image.size[0] > image.size[1]: + image = image.transpose(Image.TRANSPOSE) + image = mx.nd.expand_dims(mx.nd.array(image), axis=2) + target = image.copy() + for aug in self.input_aug: + image = aug(image) + for aug in self.target_aug: + target = aug(target) + data.append(image) + label.append(target) + + data = mx.nd.concat(*[mx.nd.expand_dims(d, axis=0) for d in data], dim=0) + label = mx.nd.concat(*[mx.nd.expand_dims(d, axis=0) for d in label], dim=0) + data = [mx.nd.transpose(data, axes=(0, 3, 1, 2)).astype('float32')/255] + label = [mx.nd.transpose(label, axes=(0, 3, 1, 2)).astype('float32')/255] + + return mx.io.DataBatch(data=data, label=label) + else: + raise StopIteration + + def reset(self): + self.count = 0 + random.shuffle(self.filenames) + +def get_model(model, ctx, opt): + """Model initialization.""" + kwargs = {'ctx': ctx, 'pretrained': opt.use_pretrained, 'classes': classes} + if model.startswith('resnet'): + kwargs['thumbnail'] = opt.use_thumbnail + elif model.startswith('vgg'): + kwargs['batch_norm'] = opt.batch_norm + + net = models.get_model(model, **kwargs) + if opt.resume: + net.load_parameters(opt.resume) + elif not opt.use_pretrained: + if model in ['alexnet']: + net.initialize(mx.init.Normal()) + else: + net.initialize(mx.init.Xavier(magnitude=2)) + net.cast(opt.dtype) + return net + +net = get_model(opt.model, context, opt) + +def get_data_iters(dataset, batch_size, opt): + """get dataset iterators""" + if dataset == 'mnist': + train_data, val_data = get_mnist_iterator(batch_size, (1, 28, 28), + num_parts=kv.num_workers, part_index=kv.rank) + elif dataset == 'cifar10': + train_data, val_data = get_cifar10_iterator(batch_size, (3, 32, 32), + num_parts=kv.num_workers, part_index=kv.rank) + elif dataset == 'imagenet': + shape_dim = 299 if model_name == 'inceptionv3' else 224 + + if not opt.data_dir: + raise ValueError('Dir containing raw images in train/val is required for imagenet.' + 'Please specify "--data-dir"') + + train_data, val_data = get_imagenet_iterator(opt.data_dir, batch_size, + opt.num_workers, shape_dim, opt.dtype) + elif dataset == 'caltech101': + train_data, val_data = get_caltech101_iterator(batch_size, opt.num_workers, opt.dtype) + elif dataset == 'dummy': + shape_dim = 299 if model_name == 'inceptionv3' else 224 + train_data, val_data = dummy_iterator(batch_size, (3, shape_dim, shape_dim)) + return train_data, val_data + +def test(ctx, val_data): + metric.reset() + val_data.reset() + for batch in val_data: + data = gluon.utils.split_and_load(batch.data[0].astype(opt.dtype, copy=False), + ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0].astype(opt.dtype, copy=False), + ctx_list=ctx, batch_axis=0) + outputs = [net(X) for X in data] + metric.update(label, outputs) + return metric.get() + +def update_learning_rate(lr, trainer, epoch, ratio, steps): + """Set the learning rate to the initial value decayed by ratio every N epochs.""" + new_lr = lr * (ratio ** int(np.sum(np.array(steps) < epoch))) + trainer.set_learning_rate(new_lr) + return trainer + +def save_checkpoint(epoch, top1, best_acc): + if opt.save_frequency and (epoch + 1) % opt.save_frequency == 0: + fname = os.path.join(opt.prefix, '%s_%d_acc_%.4f.params' % (opt.model, epoch, top1)) + net.save_parameters(fname) + logger.info('[Epoch %d] Saving checkpoint to %s with Accuracy: %.4f', epoch, fname, top1) + if top1 > best_acc[0]: + best_acc[0] = top1 + fname = os.path.join(opt.prefix, '%s_best.params' % (opt.model)) + net.save_parameters(fname) + logger.info('[Epoch %d] Saving checkpoint to %s with Accuracy: %.4f', epoch, fname, top1) + +def train(opt, ctx): + if isinstance(ctx, mx.Context): + ctx = [ctx] + + train_data, val_data = get_data_iters(dataset, batch_size, opt) + net.collect_params().reset_ctx(ctx) + trainer = gluon.Trainer(net.collect_params(), 'sgd', + optimizer_params={'learning_rate': opt.lr, + 'wd': opt.wd, + 'momentum': opt.momentum, + 'multi_precision': True}, + kvstore=kv) + loss = gluon.loss.SoftmaxCrossEntropyLoss() + + total_time = 0 + num_epochs = 0 + best_acc = [0] + for epoch in range(opt.start_epoch, opt.epochs): + trainer = update_learning_rate(opt.lr, trainer, epoch, opt.lr_factor, lr_steps) + tic = time.time() + train_data.reset() + metric.reset() + btic = time.time() + for i, batch in enumerate(train_data): + data = gluon.utils.split_and_load(batch.data[0].astype(opt.dtype), ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0].astype(opt.dtype), ctx_list=ctx, batch_axis=0) + outputs = [] + Ls = [] + with ag.record(): + for x, y in zip(data, label): + z = net(x) + L = loss(z, y) + # store the loss and do backward after we have done forward + # on all GPUs for better speed on multiple GPUs. + Ls.append(L) + outputs.append(z) + ag.backward(Ls) + trainer.step(batch.data[0].shape[0]) + metric.update(label, outputs) + if opt.log_interval and not (i+1)%opt.log_interval: + name, acc = metric.get() + logger.info('Epoch[%d] Batch [%d]\tSpeed: %f samples/sec\t%s=%f, %s=%f'%( + epoch, i, batch_size/(time.time()-btic), name[0], acc[0], name[1], acc[1])) + btic = time.time() + + epoch_time = time.time()-tic + + # First epoch will usually be much slower than the subsequent epics, + # so don't factor into the average + if num_epochs > 0: + total_time = total_time + epoch_time + num_epochs = num_epochs + 1 + + name, acc = metric.get() + logger.info('[Epoch %d] training: %s=%f, %s=%f'%(epoch, name[0], acc[0], name[1], acc[1])) + logger.info('[Epoch %d] time cost: %f'%(epoch, epoch_time)) + name, val_acc = test(ctx, val_data) + logger.info('[Epoch %d] validation: %s=%f, %s=%f'%(epoch, name[0], val_acc[0], name[1], val_acc[1])) + + # save model if meet requirements + save_checkpoint(epoch, val_acc[0], best_acc) + if num_epochs > 1: + print('Average epoch time: {}'.format(float(total_time)/(num_epochs - 1))) + +def main(): + if opt.builtin_profiler > 0: + profiler.set_config(profile_all=True, aggregate_stats=True) + profiler.set_state('run') + if opt.mode == 'symbolic': + data = mx.sym.var('data') + if opt.dtype == 'float16': + data = mx.sym.Cast(data=data, dtype=np.float16) + out = net(data) + if opt.dtype == 'float16': + out = mx.sym.Cast(data=out, dtype=np.float32) + softmax = mx.sym.SoftmaxOutput(out, name='softmax') + mod = mx.mod.Module(softmax, context=context) + train_data, val_data = get_data_iters(dataset, batch_size, opt) + mod.fit(train_data, + eval_data=val_data, + num_epoch=opt.epochs, + kvstore=kv, + batch_end_callback = mx.callback.Speedometer(batch_size, max(1, opt.log_interval)), + epoch_end_callback = mx.callback.do_checkpoint('image-classifier-%s'% opt.model), + optimizer = 'sgd', + optimizer_params = {'learning_rate': opt.lr, 'wd': opt.wd, 'momentum': opt.momentum, 'multi_precision': True}, + initializer = mx.init.Xavier(magnitude=2)) + mod.save_parameters('image-classifier-%s-%d-final.params'%(opt.model, opt.epochs)) + else: + if opt.mode == 'hybrid': + net.hybridize() + train(opt, context) + if opt.builtin_profiler > 0: + profiler.set_state('stop') + print(profiler.dumps()) + +if __name__ == '__main__': + if opt.profile: + import hotshot, hotshot.stats + prof = hotshot.Profile('image-classifier-%s-%s.prof'%(opt.model, opt.mode)) + prof.runcall(main) + prof.close() + stats = hotshot.stats.load('image-classifier-%s-%s.prof'%(opt.model, opt.mode)) + stats.strip_dirs() + stats.sort_stats('cumtime', 'calls') + stats.print_stats() + else: + main() diff --git a/dev-support/mini-submarine/submarine/mnist_distributed.py b/dev-support/mini-submarine/submarine/mnist_distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..dfa0beb5b281a8db8e625779dd9da3141149e702 --- /dev/null +++ b/dev-support/mini-submarine/submarine/mnist_distributed.py @@ -0,0 +1,275 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""A deep MNIST classifier using convolutional layers. + +This example was adapted from +https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist_deep.py. + +Each worker reads the full MNIST dataset and asynchronously trains a CNN with dropout and using the Adam optimizer, +updating the model parameters on shared parameter servers. + +The current training accuracy is printed out after every 100 steps. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.examples.tutorials.mnist import input_data + +import json +import logging +import os +import sys + +import tensorboard.program as tb_program +import tensorflow as tf +# import submarine + +# Environment variable containing port to launch TensorBoard on, set by TonY. +TB_PORT_ENV_VAR = 'TB_PORT' + +# mnist data url +tf.flags.DEFINE_string('mnist_data_url', '', + 'Url for mnist handwritten digits dataset') + +# Input/output directories +tf.flags.DEFINE_string('data_dir', '/tmp/tensorflow/mnist/input_data', + 'Directory for storing input data') +tf.flags.DEFINE_string('working_dir', '/tmp/tensorflow/mnist/working_dir', + 'Directory under which events and output will be ' + 'stored (in separate subdirectories).') + +# Training parameters +tf.flags.DEFINE_integer("steps", 1500, + "The number of training steps to execute.") +tf.flags.DEFINE_integer("batch_size", 64, "The batch size per step.") + +FLAGS = tf.flags.FLAGS + + +def deepnn(x): + """deepnn builds the graph for a deep net for classifying digits. + + Args: + x: an input tensor with the dimensions (N_examples, 784), where 784 is the + number of pixels in a standard MNIST image. + + Returns: + A tuple (y, keep_prob). y is a tensor of shape (N_examples, 10), with values + equal to the logits of classifying the digit into one of 10 classes (the + digits 0-9). keep_prob is a scalar placeholder for the probability of + dropout. + """ + # Reshape to use within a convolutional neural net. + # Last dimension is for "features" - there is only one here, since images are + # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc. + with tf.name_scope('reshape'): + x_image = tf.reshape(x, [-1, 28, 28, 1]) + + # First convolutional layer - maps one grayscale image to 32 feature maps. + with tf.name_scope('conv1'): + W_conv1 = weight_variable([5, 5, 1, 32]) + b_conv1 = bias_variable([32]) + h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) + + # Pooling layer - downsamples by 2X. + with tf.name_scope('pool1'): + h_pool1 = max_pool_2x2(h_conv1) + + # Second convolutional layer -- maps 32 feature maps to 64. + with tf.name_scope('conv2'): + W_conv2 = weight_variable([5, 5, 32, 64]) + b_conv2 = bias_variable([64]) + h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) + + # Second pooling layer. + with tf.name_scope('pool2'): + h_pool2 = max_pool_2x2(h_conv2) + + # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image + # is down to 7x7x64 feature maps -- maps this to 1024 features. + with tf.name_scope('fc1'): + W_fc1 = weight_variable([7 * 7 * 64, 1024]) + b_fc1 = bias_variable([1024]) + + h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) + h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) + + # Dropout - controls the complexity of the model, prevents co-adaptation of + # features. + with tf.name_scope('dropout'): + keep_prob = tf.placeholder(tf.float32) + h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) + + # Map the 1024 features to 10 classes, one for each digit + with tf.name_scope('fc2'): + W_fc2 = weight_variable([1024, 10]) + b_fc2 = bias_variable([10]) + + y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 + return y_conv, keep_prob + + +def conv2d(x, W): + """conv2d returns a 2d convolution layer with full stride.""" + return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') + + +def max_pool_2x2(x): + """max_pool_2x2 downsamples a feature map by 2X.""" + return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], + strides=[1, 2, 2, 1], padding='SAME') + + +def weight_variable(shape): + """weight_variable generates a weight variable of a given shape.""" + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + + +def bias_variable(shape): + """bias_variable generates a bias variable of a given shape.""" + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + + +def create_model(): + """Creates our model and returns the target nodes to be run or populated""" + # Create the model + x = tf.placeholder(tf.float32, [None, 784]) + + # Define loss and optimizer + y_ = tf.placeholder(tf.int64, [None]) + + # Build the graph for the deep net + y_conv, keep_prob = deepnn(x) + + with tf.name_scope('loss'): + cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, + logits=y_conv) + cross_entropy = tf.reduce_mean(cross_entropy) + + global_step = tf.train.get_or_create_global_step() + with tf.name_scope('adam_optimizer'): + train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy, + global_step=global_step) + + with tf.name_scope('accuracy'): + correct_prediction = tf.equal(tf.argmax(y_conv, 1), y_) + correct_prediction = tf.cast(correct_prediction, tf.float32) + accuracy = tf.reduce_mean(correct_prediction) + + tf.summary.scalar('cross_entropy_loss', cross_entropy) + tf.summary.scalar('accuracy', accuracy) + + merged = tf.summary.merge_all() + + return x, y_, keep_prob, global_step, train_step, accuracy, merged + + +def start_tensorboard(logdir): + tb = tb_program.TensorBoard() + port = int(os.getenv(TB_PORT_ENV_VAR, 6006)) + tb.configure(logdir=logdir, port=port) + tb.launch() + logging.info("Starting TensorBoard with --logdir=%s" % logdir) + + +def main(_): + logging.getLogger().setLevel(logging.INFO) + + cluster_spec_str = os.environ["CLUSTER_SPEC"] + cluster_spec = json.loads(cluster_spec_str) + ps_hosts = cluster_spec['ps'] + worker_hosts = cluster_spec['worker'] + + # Create a cluster from the parameter server and worker hosts. + cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts}) + + # Create and start a server for the local task. + job_name = os.environ["JOB_NAME"] + task_index = int(os.environ["TASK_INDEX"]) + server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) + + if job_name == "ps": + server.join() + elif job_name == "worker": + # Create our model graph. Assigns ops to the local worker by default. + with tf.device(tf.train.replica_device_setter( + worker_device="/job:worker/task:%d" % task_index, + cluster=cluster)): + features, labels, keep_prob, global_step, train_step, accuracy, \ + merged = create_model() + + if task_index is 0: # chief worker + tf.gfile.MakeDirs(FLAGS.working_dir) + start_tensorboard(FLAGS.working_dir) + + # The StopAtStepHook handles stopping after running given steps. + hooks = [tf.train.StopAtStepHook(num_steps=FLAGS.steps)] + + # Filter all connections except that between ps and this worker to + # avoid hanging issues when one worker finishes. We are using + # asynchronous training so there is no need for the workers to + # communicate. + config_proto = tf.ConfigProto( + device_filters=['/job:ps', '/job:worker/task:%d' % task_index]) + + with tf.train.MonitoredTrainingSession(master=server.target, + is_chief=(task_index == 0), + checkpoint_dir=FLAGS.working_dir, + hooks=hooks, + config=config_proto) as sess: + # Import data + logging.info('Extracting and loading input data...') + # Use a different data dir name to workaround "file already exists issue" when downloading dataset in the same single node + if FLAGS.mnist_data_url == '': + logging.info('Getting mnist data from default url') + mnist = input_data.read_data_sets(FLAGS.data_dir + str(task_index)) + else: + logging.info('Getting mnist data from ' + FLAGS.mnist_data_url) + mnist = input_data.read_data_sets(FLAGS.data_dir + str(task_index), + source_url=FLAGS.mnist_data_url) + + # Train + logging.info('Starting training') + i = 0 + while not sess.should_stop(): + # Before use submarine-sdk, start Mysql server first + # submarine.log_param("batch_size", FLAGS.batch_size) + batch = mnist.train.next_batch(FLAGS.batch_size) + if i % 100 == 0: + step, _, train_accuracy = sess.run( + [global_step, train_step, accuracy], + feed_dict={features: batch[0], labels: batch[1], + keep_prob: 1.0}) + logging.info('Step %d, training accuracy: %g' % ( + step, train_accuracy)) + # Before use submarine-sdk, start Mysql server first + # submarine.log_metric("accuracy", train_accuracy, i) + else: + sess.run([global_step, train_step], + feed_dict={features: batch[0], labels: batch[1], + keep_prob: 0.5}) + i += 1 + + logging.info('Done training!') + sys.exit() + + +if __name__ == '__main__': + tf.app.run() diff --git a/dev-support/mini-submarine/submarine/mnist_distributed_tf2.py b/dev-support/mini-submarine/submarine/mnist_distributed_tf2.py new file mode 100644 index 0000000000000000000000000000000000000000..2f880e6f5b814924aff063b2964c5888ab13e7b0 --- /dev/null +++ b/dev-support/mini-submarine/submarine/mnist_distributed_tf2.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +import tensorflow as tf +import tensorflow_datasets as tfds + +BATCH_SIZE = 64 +BUFFER_SIZE = 10000 +LEARNING_RATE = 1e-4 + +def get_task_name(): + cluster_spec = os.environ.get("CLUSTER_SPEC", None) + task_name = '' + if cluster_spec: + cluster_spec = json.loads(cluster_spec) + job_index = os.environ["TASK_INDEX"] + job_name = os.environ["JOB_NAME"] + task_name = job_name + '_' + job_index + + return task_name + +def input_fn(mode, input_context=None): + datasets, info = tfds.load(name='mnist', + data_dir='/tmp/' + get_task_name() + '/data', + with_info=True, + as_supervised=True) + + mnist_dataset = (datasets['train'] if mode == tf.estimator.ModeKeys.TRAIN + else datasets['test']) + + def scale(image, label): + image = tf.cast(image, tf.float32) + image /= 255 + return image, label + + if input_context: + mnist_dataset = mnist_dataset.shard( + input_context.num_input_pipelines, + input_context.input_pipeline_id) + + return mnist_dataset.map(scale).cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE) + +def model_fn(features, labels, mode): + model = tf.keras.Sequential([ + tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)), + tf.keras.layers.MaxPooling2D(), + tf.keras.layers.Flatten(), + tf.keras.layers.Dense(64, activation='relu'), + tf.keras.layers.Dense(10) + ]) + + logits = model(features, training=False) + + if mode == tf.estimator.ModeKeys.PREDICT: + predictions = {'logits': logits} + return tf.estimator.EstimatorSpec(labels=labels, predictions=predictions) + + optimizer = tf.compat.v1.train.GradientDescentOptimizer( + learning_rate=LEARNING_RATE) + loss = tf.keras.losses.SparseCategoricalCrossentropy( + from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(labels, logits) + loss = tf.reduce_sum(loss) * (1. / BATCH_SIZE) + if mode == tf.estimator.ModeKeys.EVAL: + return tf.estimator.EstimatorSpec(mode, loss=loss) + + return tf.estimator.EstimatorSpec( + mode=mode, + loss=loss, + train_op=optimizer.minimize( + loss, tf.compat.v1.train.get_or_create_global_step())) + +if __name__ == '__main__': + strategy = tf.distribute.experimental.ParameterServerStrategy() + config = tf.estimator.RunConfig(train_distribute=strategy) + estimator = tf.estimator.Estimator( + model_fn=model_fn, + model_dir='/tmp/model', + config=config) + train_spec = tf.estimator.TrainSpec(input_fn) + eval_spec = tf.estimator.EvalSpec(input_fn) + tf.estimator.train_and_evaluate( + estimator, + train_spec, + eval_spec) diff --git a/dev-support/mini-submarine/submarine/pytorch_mnist_distributed.py b/dev-support/mini-submarine/submarine/pytorch_mnist_distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..667603aefb9570c2bf1a044e35679af22ebb7e45 --- /dev/null +++ b/dev-support/mini-submarine/submarine/pytorch_mnist_distributed.py @@ -0,0 +1,242 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""A deep MNIST classifier using convolutional layers. + +This example was adapted from +https://pytorch.org/docs/master/distributed.html +https://pytorch.org/tutorials/intermediate/dist_tuto.html +https://github.com/narumiruna/pytorch-distributed-example/blob/master/mnist/main.py + +Each worker reads the full MNIST dataset and asynchronously trains a CNN with dropout and using the Adam optimizer, +updating the model parameters on shared parameter servers. + +The current training accuracy is printed out after every 100 steps. +""" + +from __future__ import division, print_function + +import argparse + +import os +import torch +import torch.nn.functional as F +from torch import distributed, nn +from torch.utils import data +from torch.utils.data.distributed import DistributedSampler +from torchvision import datasets, transforms + + +class AverageMeter(object): + + def __init__(self): + self.sum = 0 + self.count = 0 + + def update(self, value, number): + self.sum += value * number + self.count += number + + @property + def average(self): + return self.sum / self.count + + +class AccuracyMeter(object): + + def __init__(self): + self.correct = 0 + self.count = 0 + + def update(self, output, label): + predictions = output.data.argmax(dim=1) + correct = predictions.eq(label.data).sum().item() + + self.correct += correct + self.count += output.size(0) + + @property + def accuracy(self): + return self.correct / self.count + + +class Trainer(object): + + def __init__(self, net, optimizer, train_loader, test_loader, device): + self.net = net + self.optimizer = optimizer + self.train_loader = train_loader + self.test_loader = test_loader + self.device = device + + def train(self): + train_loss = AverageMeter() + train_acc = AccuracyMeter() + + self.net.train() + + for data, label in self.train_loader: + data = data.to(self.device) + label = label.to(self.device) + + output = self.net(data) + loss = F.cross_entropy(output, label) + + self.optimizer.zero_grad() + loss.backward() + # average the gradients + self.average_gradients() + self.optimizer.step() + + train_loss.update(loss.item(), data.size(0)) + train_acc.update(output, label) + + return train_loss.average, train_acc.accuracy + + def evaluate(self): + test_loss = AverageMeter() + test_acc = AccuracyMeter() + + self.net.eval() + + with torch.no_grad(): + for data, label in self.test_loader: + data = data.to(self.device) + label = label.to(self.device) + + output = self.net(data) + loss = F.cross_entropy(output, label) + + test_loss.update(loss.item(), data.size(0)) + test_acc.update(output, label) + + return test_loss.average, test_acc.accuracy + + def average_gradients(self): + world_size = distributed.get_world_size() + + for p in self.net.parameters(): + group = distributed.new_group(ranks=list(range(world_size))) + + tensor = p.grad.data.cpu() + + distributed.all_reduce( + tensor, op=distributed.reduce_op.SUM, group=group) + + tensor /= float(world_size) + + p.grad.data = tensor.to(self.device) + + +class Net(nn.Module): + + def __init__(self): + super(Net, self).__init__() + self.fc = nn.Linear(784, 10) + + def forward(self, x): + return self.fc(x.view(x.size(0), -1)) + + +def get_dataloader(root, batch_size): + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.13066047740239478,), (0.3081078087569972,)) + ]) + + train_set = datasets.MNIST( + root, train=True, transform=transform, download=True) + sampler = DistributedSampler(train_set) + + train_loader = data.DataLoader( + train_set, + batch_size=batch_size, + shuffle=(sampler is None), + sampler=sampler) + + test_loader = data.DataLoader( + datasets.MNIST(root, train=False, transform=transform, download=True), + batch_size=batch_size, + shuffle=False) + + return train_loader, test_loader + + +def solve(args): + device = torch.device('cuda' if args.cuda else 'cpu') + + net = Net().to(device) + + optimizer = torch.optim.Adam(net.parameters(), lr=args.learning_rate) + + train_loader, test_loader = get_dataloader(args.root, args.batch_size) + + trainer = Trainer(net, optimizer, train_loader, test_loader, device) + + for epoch in range(1, args.epochs + 1): + train_loss, train_acc = trainer.train() + test_loss, test_acc = trainer.evaluate() + + print( + 'Epoch: {}/{},'.format(epoch, args.epochs), + 'train loss: {:.6f}, train acc: {:.6f}, test loss: {:.6f}, test acc: {:.6f}.'. + format(train_loss, train_acc, test_loss, test_acc)) + + +def init_process(args): + distributed.init_process_group( + backend=args.backend, + init_method=args.init_method, + rank=args.rank, + world_size=args.world_size) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--backend', + type=str, + default='tcp', + help='Name of the backend to use.') + parser.add_argument( + '--init-method', + '-i', + type=str, + default=os.environ.get('INIT_METHOD', 'tcp://127.0.0.1:23456'), + help='URL specifying how to initialize the package.') + parser.add_argument( + '--rank', '-r', + type=int, + default=int(os.environ.get('RANK')), + help='Rank of the current process.') + parser.add_argument( + '--world-size', + '-s', + type=int, + default=int(os.environ.get('WORLD')), + help='Number of processes participating in the job.') + parser.add_argument('--epochs', type=int, default=20) + parser.add_argument('--no-cuda', action='store_true') + parser.add_argument('--learning-rate', '-lr', type=float, default=1e-3) + parser.add_argument('--root', type=str, default='data') + parser.add_argument('--batch-size', type=int, default=128) + args = parser.parse_args() + args.cuda = torch.cuda.is_available() and not args.no_cuda + print(args) + + init_process(args) + solve(args) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/dev-support/mini-submarine/submarine/run_customized_submarine-all_mnist.sh b/dev-support/mini-submarine/submarine/run_customized_submarine-all_mnist.sh new file mode 100755 index 0000000000000000000000000000000000000000..d08a7b981b205336019f3ed65d393e6b2e650c39 --- /dev/null +++ b/dev-support/mini-submarine/submarine/run_customized_submarine-all_mnist.sh @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash + +# Need to modify environment variables based on version +while [ $# -gt 0 ]; do + case "$1" in + --debug*) + DEBUG=$1 + shift + ;; + *) + break + ;; + esac +done + +DEBUG_PORT=8000 +if [ "$DEBUG" ]; then + JAVA_CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=${DEBUG_PORT}" +else + JAVA_CMD="java" +fi + +while getopts 'd:' OPT; do + case $OPT in + d) + DATA_URL="$OPTARG";; + esac +done +shift $(($OPTIND - 1)) + +if [[ -n "$DATA_URL" ]]; then + WORKER_CMD="myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode --mnist_data_url ${DATA_URL}" +else + WORKER_CMD="myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" +fi + +SUBMARINE_VERSION=${SUBMARINE_VER:-"0.5.0"} + +HADOOP_VERSION=2.9 + +${JAVA_CMD} -cp /tmp/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar:/usr/local/hadoop/etc/hadoop \ +org.apache.submarine.client.cli.Cli job run --name tf-job-002 \ +--framework tensorflow \ +--verbose \ +--input_path "" \ +--num_workers 2 \ +--worker_resources memory=1G,vcores=1 \ +--num_ps 1 \ +--ps_resources memory=1G,vcores=1 \ +--worker_launch_cmd "${WORKER_CMD}" \ +--ps_launch_cmd "myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" \ +--insecure \ +--conf tony.containers.resources=/home/yarn/submarine/myvenv.zip#archive,/home/yarn/submarine/mnist_distributed.py,/tmp/submarine-dist-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar diff --git a/dev-support/mini-submarine/submarine/run_submarine_mnist_tf2_tony.sh b/dev-support/mini-submarine/submarine/run_submarine_mnist_tf2_tony.sh new file mode 100755 index 0000000000000000000000000000000000000000..b12efbff9809c93c91e2d6097b0cbdf601bbc9f0 --- /dev/null +++ b/dev-support/mini-submarine/submarine/run_submarine_mnist_tf2_tony.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +while [ $# -gt 0 ]; do + case "$1" in + --debug*) + DEBUG=$1 + if [ -n "$2" ]; then + DEBUG_PORT=$2 + shift + fi + shift + ;; + *) + break + ;; + esac +done + +if [ "$DEBUG" ]; then + if [ -z "$DEBUG_PORT" ]; then + DEBUG_PORT=8000 + fi + JAVA_CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=${DEBUG_PORT}" +else + JAVA_CMD="java" +fi + +SUBMARINE_VERSION=0.5.0 +HADOOP_VERSION=2.9 +SUBMARINE_PATH=/opt/submarine-current +HADOOP_CONF_PATH=/usr/local/hadoop/etc/hadoop + +${JAVA_CMD} -cp "$("${HADOOP_COMMON_HOME}"/bin/hadoop classpath --glob)":${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-"${HADOOP_VERSION}".jar:${HADOOP_CONF_PATH} \ + org.apache.submarine.client.cli.Cli job run --name tf2-job-001 \ + --framework tensorflow \ + --input_path "" \ + --num_workers 2 \ + --worker_resources memory=1G,vcores=1 \ + --num_ps 1 \ + --ps_resources memory=1G,vcores=1 \ + --worker_launch_cmd "tf2-venv.zip/tf2-venv/bin/python mnist_distributed_tf2.py" \ + --ps_launch_cmd "tf2-venv.zip/tf2-venv/bin/python mnist_distributed_tf2.py" \ + --insecure \ + --verbose \ + --conf tony.containers.resources=/home/yarn/submarine/tf2-venv.zip#archive,/home/yarn/submarine/mnist_distributed_tf2.py,${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-"${HADOOP_VERSION}".jar diff --git a/dev-support/mini-submarine/submarine/run_submarine_mnist_tony.sh b/dev-support/mini-submarine/submarine/run_submarine_mnist_tony.sh new file mode 100755 index 0000000000000000000000000000000000000000..272ac320d2a74f233b74955650a700e8e109e3df --- /dev/null +++ b/dev-support/mini-submarine/submarine/run_submarine_mnist_tony.sh @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash + +# Below are configurable variables, please adapt base on your local environment. +# Version of submarine jar +SUBMARINE_VERSION=${SUBMARINE_VER:-"0.5.0"} + +# Version of affiliated Hadoop version for this Submarine jar. +SUBMARINE_HADOOP_VERSION=2.9 + +# Path to the submarine jars. +SUBMARINE_PATH=/opt/submarine-current + +# Similar to HADOOP_CONF_DIR, location of the Hadoop configuration directory +HADOOP_CONF_PATH=/usr/local/hadoop/etc/hadoop + +# Path to the MNIST example. +MNIST_PATH=/home/yarn/submarine + +while [ $# -gt 0 ]; do + case "$1" in + --debug*) + DEBUG=$1 + shift + ;; + *) + break + ;; + esac +done + +DEBUG_PORT=8000 +if [ "$DEBUG" ]; then + JAVA_CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=${DEBUG_PORT}" +else + JAVA_CMD="java" +fi + +while getopts 'd:c' OPT; do + case $OPT in + d) + DATA_URL="$OPTARG";; + c) + USE_DOCKER=1;; + esac +done +shift $(($OPTIND - 1)) + +if [[ -n "$DATA_URL" ]]; then + WORKER_CMD="venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode --mnist_data_url ${DATA_URL}" +else + WORKER_CMD="venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" +fi + +if [[ -n "$USE_DOCKER" ]]; then + WORKER_CMD="/opt/$WORKER_CMD" + # tony-mnist-tf-1.13.1:0.0.1 is built from the Dockerfile.tony.tf.mnist.tf_1.13.1 under docs/helper/docker/tensorflow/mnist + DOCKER_CONF="--conf tony.docker.containers.image=tony-mnist-tf-1.13.1:0.0.1 --conf tony.docker.enabled=true" +else + WORKER_CMD="myvenv.zip/$WORKER_CMD" +fi + +${JAVA_CMD} -cp $(${HADOOP_COMMON_HOME}/bin/hadoop classpath --glob):${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar:${HADOOP_CONF_PATH} \ + org.apache.submarine.client.cli.Cli job run --name tf-job-001 \ + --framework tensorflow \ + --verbose \ + --input_path "" \ + --num_workers 2 \ + --worker_resources memory=1G,vcores=1 \ + --num_ps 1 \ + --ps_resources memory=1G,vcores=1 \ + --worker_launch_cmd "${WORKER_CMD}" \ + --ps_launch_cmd "myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" \ + --insecure \ + --conf tony.containers.resources=${MNIST_PATH}/myvenv.zip#archive,${MNIST_PATH}/mnist_distributed.py,${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar \ + $DOCKER_CONF diff --git a/dev-support/mini-submarine/submarine/run_submarine_mnist_tony_rpc.sh b/dev-support/mini-submarine/submarine/run_submarine_mnist_tony_rpc.sh new file mode 100644 index 0000000000000000000000000000000000000000..602e9d8d494263b6f0afdc739596630f4167c3ad --- /dev/null +++ b/dev-support/mini-submarine/submarine/run_submarine_mnist_tony_rpc.sh @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +while [ $# -gt 0 ]; do + case "$1" in + --debug*) + DEBUG=$1 + shift + ;; + *) + break + ;; + esac +done + +DEBUG_PORT=8000 +if [ "$DEBUG" ]; then + JAVA_CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=${DEBUG_PORT}" +else + JAVA_CMD="java" +fi + +while getopts 'd:' OPT; do + case $OPT in + d) + DATA_URL="$OPTARG";; + esac +done +shift $(($OPTIND - 1)) + +if [[ -n "$DATA_URL" ]]; then + WORKER_CMD="myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode --mnist_data_url ${DATA_URL}" +else + WORKER_CMD="myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" +fi + +SUBMARINE_VERSION=${SUBMARINE_VER:-"0.5.0"} + +HADOOP_VERSION=2.9 + +${JAVA_CMD} -cp /opt/submarine-dist-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar:/usr/local/hadoop/etc/hadoop:/opt/submarine-dist-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}/conf \ + org.apache.submarine.client.cli.Cli job run --name tf-job-001 \ + --framework tensorflow \ + --verbose \ + --input_path "" \ + --num_workers 2 \ + --worker_resources memory=1G,vcores=1 \ + --num_ps 1 \ + --ps_resources memory=1G,vcores=1 \ + --worker_launch_cmd "${WORKER_CMD}" \ + --ps_launch_cmd "myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" \ + --insecure \ + --conf tony.containers.resources=/home/yarn/submarine/myvenv.zip#archive,/home/yarn/submarine/mnist_distributed.py,/opt/submarine-dist-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar diff --git a/dev-support/mini-submarine/submarine/run_submarine_mxnet_cifar10_tony.sh b/dev-support/mini-submarine/submarine/run_submarine_mxnet_cifar10_tony.sh new file mode 100755 index 0000000000000000000000000000000000000000..4dec8687e9c6340f401c8a138a0475f24635bdcc --- /dev/null +++ b/dev-support/mini-submarine/submarine/run_submarine_mxnet_cifar10_tony.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +while [ $# -gt 0 ]; do + case "$1" in + --debug*) + DEBUG=$1 + if [ -n "$2" ]; then + DEBUG_PORT=$2 + shift + fi + shift + ;; + *) + break + ;; + esac +done + +if [ "$DEBUG" ]; then + if [ -z "$DEBUG_PORT" ]; then + DEBUG_PORT=8000 + fi + JAVA_CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=${DEBUG_PORT}" +else + JAVA_CMD="java" +fi + +SUBMARINE_VERSION=0.5.0 +HADOOP_VERSION=2.9 +SUBMARINE_PATH=/opt/submarine-current +HADOOP_CONF_PATH=/usr/local/hadoop/etc/hadoop + +${JAVA_CMD} -cp ${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar:${HADOOP_CONF_PATH} \ + org.apache.submarine.client.cli.Cli job run \ + --name mx-job-001 \ + --framework mxnet \ + --input_path "" \ + --num_ps 1 \ + --ps_resources memory=1G,vcores=1 \ + --ps_launch_cmd "myvenv.zip/venv/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --num_workers 2 \ + --worker_resources memory=2G,vcores=1 \ + --worker_launch_cmd "myvenv.zip/venv/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --num_schedulers 1 \ + --scheduler_resources memory=1G,vcores=1 \ + --scheduler_launch_cmd "myvenv.zip/venv/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --insecure \ + --verbose \ + --conf tony.containers.resources=/home/yarn/submarine/myvenv.zip#archive,/home/yarn/submarine/image_classification.py,${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar diff --git a/dev-support/mini-submarine/submarine/run_submarine_pytorch_mnist_tony.sh b/dev-support/mini-submarine/submarine/run_submarine_pytorch_mnist_tony.sh new file mode 100755 index 0000000000000000000000000000000000000000..b57dd566971781e32b15c19c88ed102423dae001 --- /dev/null +++ b/dev-support/mini-submarine/submarine/run_submarine_pytorch_mnist_tony.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +while [ $# -gt 0 ]; do + case "$1" in + --debug*) + DEBUG=$1 + if [ -n "$2" ]; then + DEBUG_PORT=$2 + shift + fi + shift + ;; + *) + break + ;; + esac +done + +if [ "$DEBUG" ]; then + if [ -z "$DEBUG_PORT" ]; then + DEBUG_PORT=8000 + fi + JAVA_CMD="java -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=${DEBUG_PORT}" +else + JAVA_CMD="java" +fi + +SUBMARINE_VERSION=0.5.0 +HADOOP_VERSION=2.9 +SUBMARINE_PATH=/opt/submarine-current +HADOOP_CONF_PATH=/usr/local/hadoop/etc/hadoop +MNIST_PATH=/home/yarn/submarine + +${JAVA_CMD} -cp ${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar:${HADOOP_CONF_PATH} \ + org.apache.submarine.client.cli.Cli job run \ + --name pytorch-job-001 \ + --framework pytorch \ + --input_path "" \ + --num_workers 2 \ + --worker_resources memory=1G,vcores=1 \ + --worker_launch_cmd "myvenv.zip/venv/bin/python pytorch_mnist_distributed.py" \ + --insecure \ + --verbose \ + --conf tony.containers.resources=${MNIST_PATH}/myvenv.zip#archive,${MNIST_PATH}/pytorch_mnist_distributed.py,${SUBMARINE_PATH}/submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar diff --git a/dev-support/pysubmarine/Dockerfile b/dev-support/pysubmarine/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..90878f43811dde17ea0bc0f2eb6d027dc86d334b --- /dev/null +++ b/dev-support/pysubmarine/Dockerfile @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 + +RUN apt-get update && apt-get install -y --no-install-recommends apt-utils +# Install JAVA +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install Docker +RUN apt-get update && \ + apt-get -y install apt-transport-https ca-certificates curl software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + apt-get update && \ + apt-get -y install docker-ce +# So no need to mount host's /var/run/docker.sock, dockerd will create in local fs +VOLUME /var/lib/docker +VOLUME /var/lib/docker.sock + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +ENV JRE_HOME /usr/lib/jvm/java-8-openjdk-amd64/jre + +# Install user tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + git make libgtest-dev cmake wget unzip libtinfo-dev libz-dev \ + libcurl4-openssl-dev libopenblas-dev g++ sudo \ + apt-transport-https curl vim ca-certificates maven + +WORKDIR /root + +# Intstall conda +ADD install-conda.sh /usr/local +ADD bootstrap.sh /usr/local +RUN chmod 755 /usr/local/install-conda.sh && \ + bash /usr/local/install-conda.sh + +WORKDIR /workspace diff --git a/dev-support/pysubmarine/bootstrap.sh b/dev-support/pysubmarine/bootstrap.sh new file mode 100755 index 0000000000000000000000000000000000000000..d4e310544143d6dfbfdba922d5ce0f2ad5371665 --- /dev/null +++ b/dev-support/pysubmarine/bootstrap.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -euo pipefail + +COMMAND=("$@") + +getent group "${CI_BUILD_GID}" || addgroup --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}" +getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \ + --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \ + --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}" +usermod -a -G sudo "${CI_BUILD_USER}" +echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo +# shellcheck disable=SC2024 +sudo nohup dockerd --host=unix:///var/run/docker.sock > /var/log/dockerd.log 2>&1 & +sudo usermod -aG docker "${CI_BUILD_USER}" + +sudo -u "#${CI_BUILD_UID}" --preserve-env \ +HOME="${CI_BUILD_HOME}" \ +PATH="${PATH}" \ +PYTHONPATH="${PYTHONPATH}" \ +"${COMMAND[@]}" diff --git a/dev-support/pysubmarine/gen-sdk.sh b/dev-support/pysubmarine/gen-sdk.sh new file mode 100755 index 0000000000000000000000000000000000000000..b83eab1b7102a848fd4d95929b254e613121040a --- /dev/null +++ b/dev-support/pysubmarine/gen-sdk.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FWDIR="$(cd "$(dirname "$0")"; pwd)" +cd "$FWDIR" + +SUBMARINE_PROJECT_PATH="$FWDIR/../.." +SWAGGER_JAR_URL="https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/4.3.1/openapi-generator-cli-4.3.1.jar" +SWAGGER_CODEGEN_JAR="openapi-generator-cli.jar" +SWAGGER_CODEGEN_CONF="swagger_config.json" +SWAGGER_CODEGEN_FILE="openapi.json" +SDK_OUTPUT_PATH="sdk/python" + +submarine_dist_exists=$(find -L "${SUBMARINE_PROJECT_PATH}/submarine-dist/target" -name "submarine-dist-*.tar.gz") +# Build source code if the package doesn't exist. +if [[ -z "${submarine_dist_exists}" ]]; then + cd "${SUBMARINE_PROJECT_PATH}" + mvn clean package -DskipTests + cd "$FWDIR" # go back to FWDIR after packaging +fi + +echo "Generating openAPI 3.0 definition file ..." +# TODO(pingsutw): generate openapi.json without starting submarine server +bash ${SUBMARINE_PROJECT_PATH}/submarine-dist/target//submarine-dist-*/submarine-dist-*/bin/submarine-daemon.sh start getMysqlJar +sleep 5 +rm openapi.json +wget http://localhost:8080/v1/openapi.json +bash ${SUBMARINE_PROJECT_PATH}/submarine-dist/target//submarine-dist-*/submarine-dist-*/bin/submarine-daemon.sh stop + +openapi_generator_cli_exists=$(find -L "${FWDIR}" -name "openapi-generator-cli*") +if [[ -z "${openapi_generator_cli_exists}" ]]; then + echo "Downloading the swagger-codegen JAR package ..." + wget -O "${SWAGGER_CODEGEN_JAR}" "${SWAGGER_JAR_URL}" +fi + +echo "Generating Python SDK for Submarine ..." +rm -r sdk/ +java -jar ${SWAGGER_CODEGEN_JAR} generate \ + -i "${SWAGGER_CODEGEN_FILE}" \ + -g python \ + -o ${SDK_OUTPUT_PATH} \ + -c ${SWAGGER_CODEGEN_CONF} + +echo "Insert apache license at the top of file ..." +for filename in $(find ${SDK_OUTPUT_PATH}/submarine/experiment -type f); do + echo "$filename" + sed -i -e '1 e cat license-header.txt' "$filename" +done + +echo "Move Experiment API to pysubmarine" +cp -r sdk/python/submarine/experiment ${SUBMARINE_PROJECT_PATH}/submarine-sdk/pysubmarine/submarine/ + +echo "Fix Python SDK code style" +${SUBMARINE_PROJECT_PATH}/submarine-sdk/pysubmarine/github-actions/auto-format.sh diff --git a/dev-support/pysubmarine/gen-ts-sdk.sh b/dev-support/pysubmarine/gen-ts-sdk.sh new file mode 100755 index 0000000000000000000000000000000000000000..96c5437f8edb72df14bc25ebf3851848e415743a --- /dev/null +++ b/dev-support/pysubmarine/gen-ts-sdk.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FWDIR="$(cd "$(dirname "$0")"; pwd)" +cd "$FWDIR" + +SUBMARINE_PROJECT_PATH="$FWDIR/../.." +SWAGGER_JAR_URL="https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/4.3.1/openapi-generator-cli-4.3.1.jar" +SWAGGER_CODEGEN_JAR="openapi-generator-cli.jar" +SWAGGER_CODEGEN_CONF="swagger_config.json" +SWAGGER_CODEGEN_FILE="openapi.json" +SDK_OUTPUT_PATH="sdk/typescript-angular" + +submarine_dist_exists=$(find -L "${SUBMARINE_PROJECT_PATH}/submarine-dist/target" -name "submarine-dist-*.tar.gz") +# Build source code if the package doesn't exist. +if [[ -z "${submarine_dist_exists}" ]]; then + cd "${SUBMARINE_PROJECT_PATH}" + mvn clean package -DskipTests + cd "$FWDIR" # go back to FWDIR after packaging +fi + +echo "Generating openAPI 3.0 definition file ..." +# TODO(pingsutw): generate openapi.json without starting submarine server +bash ${SUBMARINE_PROJECT_PATH}/submarine-dist/target//submarine-dist-*/submarine-dist-*/bin/submarine-daemon.sh start getMysqlJar +sleep 5 +rm openapi.json +wget http://localhost:8080/v1/openapi.json +bash ${SUBMARINE_PROJECT_PATH}/submarine-dist/target//submarine-dist-*/submarine-dist-*/bin/submarine-daemon.sh stop + +openapi_generator_cli_exists=$(find -L "${FWDIR}" -name "openapi-generator-cli*") +if [[ -z "${openapi_generator_cli_exists}" ]]; then + echo "Downloading the swagger-codegen JAR package ..." + wget -O "${SWAGGER_CODEGEN_JAR}" "${SWAGGER_JAR_URL}" +fi + +echo "Generating typescript-angular SDK for Submarine ..." +rm -r sdk/ +java -jar ${SWAGGER_CODEGEN_JAR} generate \ + -i "${SWAGGER_CODEGEN_FILE}" \ + -g typescript-angular \ + -o ${SDK_OUTPUT_PATH} \ + -c ${SWAGGER_CODEGEN_CONF} diff --git a/dev-support/pysubmarine/install-conda.sh b/dev-support/pysubmarine/install-conda.sh new file mode 100644 index 0000000000000000000000000000000000000000..be6a7cd4e14e73510011d12bd33f25b773e6da37 --- /dev/null +++ b/dev-support/pysubmarine/install-conda.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +set -x + +wget https://repo.anaconda.com/archive/Anaconda3-2020.02-Linux-x86_64.sh -O "$HOME"/anaconda.sh; +bash "$HOME"/anaconda.sh -b -p /usr/bin/anaconda +export PATH="/usr/bin/anaconda/bin:$PATH" +cd "$HOME" +# Useful for debugging any issues with conda +conda info -a +if [[ -n "${PYTHON_VERSION:-}" ]]; then + conda create -q -n submarine-dev python="$PYTHON_VERSION" +else + conda create -q -n submarine-dev python=3.6 +fi + +source activate submarine-dev +python --version +pip install --upgrade pip + +# Install pysubmarine +git clone https://github.com/apache/submarine.git +cd submarine/submarine-sdk/pysubmarine +pip install . +pip install -r github-actions/lint-requirements.txt +pip install -r github-actions/test-requirements.txt diff --git a/dev-support/pysubmarine/license-header.txt b/dev-support/pysubmarine/license-header.txt new file mode 100644 index 0000000000000000000000000000000000000000..09697dce6e11859e38a9bfb321bfc72eb88b8ce6 --- /dev/null +++ b/dev-support/pysubmarine/license-header.txt @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/dev-support/pysubmarine/openapi.json b/dev-support/pysubmarine/openapi.json new file mode 100644 index 0000000000000000000000000000000000000000..9ab79b23ff87606310b5aaac3271e0ad7a0d0d7a --- /dev/null +++ b/dev-support/pysubmarine/openapi.json @@ -0,0 +1,409 @@ +{ + "openapi" : "3.0.1", + "info" : { + "title" : "Submarine Experiment API", + "description" : "The Submarine REST API allows you to create, list, and get experiments. The\nAPI is hosted under the /v1/experiment route on the Submarine server. For example,\nto list experiments on a server hosted at http://localhost:8080, access\nhttp://localhost:8080/api/v1/experiment/", + "termsOfService" : "http://swagger.io/terms/", + "contact" : { + "email" : "dev@submarine.apache.org" + }, + "license" : { + "name" : "Apache 2.0", + "url" : "http://www.apache.org/licenses/LICENSE-2.0.html" + }, + "version" : "0.5.0" + }, + "servers" : [ { + "url" : "/api" + } ], + "paths" : { + "/v1/experiment/{id}" : { + "get" : { + "tags" : [ "experiment" ], + "summary" : "Get the experiment's detailed info by id", + "operationId" : "getExperiment", + "parameters" : [ { + "name" : "id", + "in" : "path", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "default" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "$ref" : "#/components/schemas/JsonResponse" + } + } + } + }, + "404" : { + "description" : "Experiment not found" + } + } + }, + "delete" : { + "tags" : [ "experiment" ], + "summary" : "Delete the experiment", + "operationId" : "deleteExperiment", + "parameters" : [ { + "name" : "id", + "in" : "path", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "default" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "$ref" : "#/components/schemas/JsonResponse" + } + } + } + }, + "404" : { + "description" : "Experiment not found" + } + } + }, + "patch" : { + "tags" : [ "experiment" ], + "summary" : "Update the experiment in the submarine server with spec", + "operationId" : "patchExperiment", + "parameters" : [ { + "name" : "id", + "in" : "path", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "requestBody" : { + "content" : { + "application/yaml" : { + "schema" : { + "$ref" : "#/components/schemas/ExperimentSpec" + } + }, + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/ExperimentSpec" + } + } + } + }, + "responses" : { + "default" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "$ref" : "#/components/schemas/JsonResponse" + } + } + } + }, + "404" : { + "description" : "Experiment not found" + } + } + } + }, + "/v1/experiment/logs" : { + "get" : { + "tags" : [ "experiment" ], + "summary" : "List experiment's log", + "operationId" : "listLog", + "parameters" : [ { + "name" : "status", + "in" : "query", + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "default" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "$ref" : "#/components/schemas/JsonResponse" + } + } + } + } + } + } + }, + "/v1/experiment" : { + "get" : { + "tags" : [ "experiment" ], + "summary" : "List experiments", + "operationId" : "listExperiments", + "parameters" : [ { + "name" : "status", + "in" : "query", + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "default" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "$ref" : "#/components/schemas/JsonResponse" + } + } + } + } + } + }, + "post" : { + "tags" : [ "experiment" ], + "summary" : "Create an experiment", + "operationId" : "createExperiment", + "requestBody" : { + "content" : { + "application/yaml" : { + "schema" : { + "$ref" : "#/components/schemas/ExperimentSpec" + } + }, + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/ExperimentSpec" + } + } + } + }, + "responses" : { + "default" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "$ref" : "#/components/schemas/JsonResponse" + } + } + } + } + } + } + }, + "/v1/experiment/logs/{id}" : { + "get" : { + "tags" : [ "experiment" ], + "summary" : "Log experiment by id", + "operationId" : "getLog", + "parameters" : [ { + "name" : "id", + "in" : "path", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "default" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "$ref" : "#/components/schemas/JsonResponse" + } + } + } + }, + "404" : { + "description" : "Experiment not found" + } + } + } + }, + "/v1/experiment/ping" : { + "get" : { + "tags" : [ "experiment" ], + "summary" : "Ping submarine server", + "description" : "Return the Pong message for test the connectivity", + "operationId" : "ping", + "responses" : { + "200" : { + "description" : "successful operation", + "content" : { + "application/json; charset=utf-8" : { + "schema" : { + "type" : "string" + } + } + } + } + } + } + } + }, + "components" : { + "schemas" : { + "JsonResponse" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "integer", + "format" : "int32" + }, + "success" : { + "type" : "boolean" + }, + "result" : { + "type" : "object" + }, + "attributes" : { + "type" : "object", + "additionalProperties" : { + "type" : "object" + } + } + } + }, + "CodeSpec" : { + "type" : "object", + "properties" : { + "syncMode" : { + "type" : "string" + }, + "url" : { + "type" : "string" + } + } + }, + "EnvironmentSpec" : { + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + }, + "dockerImage" : { + "type" : "string" + }, + "kernelSpec" : { + "$ref" : "#/components/schemas/KernelSpec" + }, + "description" : { + "type" : "string" + }, + "image" : { + "type" : "string" + } + } + }, + "ExperimentMeta" : { + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + }, + "namespace" : { + "type" : "string" + }, + "framework" : { + "type" : "string" + }, + "cmd" : { + "type" : "string" + }, + "envVars" : { + "type" : "object", + "additionalProperties" : { + "type" : "string" + } + } + } + }, + "ExperimentSpec" : { + "type" : "object", + "properties" : { + "meta" : { + "$ref" : "#/components/schemas/ExperimentMeta" + }, + "environment" : { + "$ref" : "#/components/schemas/EnvironmentSpec" + }, + "spec" : { + "type" : "object", + "additionalProperties" : { + "$ref" : "#/components/schemas/ExperimentTaskSpec" + } + }, + "code" : { + "$ref" : "#/components/schemas/CodeSpec" + } + } + }, + "ExperimentTaskSpec" : { + "type" : "object", + "properties" : { + "replicas" : { + "type" : "integer", + "format" : "int32" + }, + "resources" : { + "type" : "string" + }, + "name" : { + "type" : "string" + }, + "image" : { + "type" : "string" + }, + "cmd" : { + "type" : "string" + }, + "envVars" : { + "type" : "object", + "additionalProperties" : { + "type" : "string" + } + }, + "cpu" : { + "type" : "string" + }, + "gpu" : { + "type" : "string" + }, + "memory" : { + "type" : "string" + } + } + }, + "KernelSpec" : { + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + }, + "channels" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "dependencies" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } + } + } +} diff --git a/dev-support/pysubmarine/run-pysubmarine-ci.sh b/dev-support/pysubmarine/run-pysubmarine-ci.sh new file mode 100755 index 0000000000000000000000000000000000000000..317e49154922752d9cb761450bdc38ecadc18704 --- /dev/null +++ b/dev-support/pysubmarine/run-pysubmarine-ci.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FWDIR="$(cd "$(dirname "$0")"; pwd)" +cd "$FWDIR" + +# shellcheck disable=SC2034 +SUBMARINE_PROJECT_PATH="$FWDIR/../.." + +# build image +echo "Start building the mini-submarine docker image..." +docker build --tag pysubmarine-ci . + +docker run --rm --pid=host \ + --privileged \ + -v "$SUBMARINE_PROJECT_PATH":/workspace \ + -e "CI_BUILD_HOME=/" \ + -e "CI_BUILD_USER=$(id -u -n)" \ + -e "CI_BUILD_UID=$(id -u)" \ + -e "CI_BUILD_GROUP=$(id -g -n)" \ + -e "CI_BUILD_GID=$(id -g)" \ + -e "PATH=/usr/bin/anaconda/envs/submarine-dev/bin:${PATH}" \ + -e "PYTHONPATH=python:/usr/bin/anaconda/envs/submarine-dev/bin/python"\ + -it pysubmarine-ci \ + /bin/bash --login /usr/local/bootstrap.sh bash diff --git a/dev-support/pysubmarine/swagger_config.json b/dev-support/pysubmarine/swagger_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d7e0772f1a7e1c20092fba94c29f581da28923cc --- /dev/null +++ b/dev-support/pysubmarine/swagger_config.json @@ -0,0 +1,5 @@ +{ + "packageName" : "submarine.experiment", + "projectName" : "submarine.experiment", + "packageVersion": "0.5.0" +} diff --git a/dev-support/style-check/lint-angular.sh b/dev-support/style-check/lint-angular.sh new file mode 100755 index 0000000000000000000000000000000000000000..e6af29b872ad0fb377fb757727454f674abc1c55 --- /dev/null +++ b/dev-support/style-check/lint-angular.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +npm install prettier@^2.0.5 + +WORKBENCH_NG=./submarine-workbench/workbench-web + +PRETTIER_ERRORS=$(./node_modules/.bin/prettier --check --trailing-comma none "$WORKBENCH_NG/src/**/*.{ts,html}" | grep "Forgot to run Prettier?") + + +if test "$PRETTIER_ERRORS"; then + echo -e "prettier checks failed at following occurrences:\n$PRETTIER_ERRORS\n" + echo -e "Please use \\033[31m"./node_modules/.bin/prettier --write --trailing-comma none "$WORKBENCH_NG/src/**/*.{ts,html}""\\033[0m to format code automatically\n" + exit 1 +else + echo -e "Checkstyle checks passed." +fi + diff --git a/dev-support/style-check/lint-java.sh b/dev-support/style-check/lint-java.sh new file mode 100755 index 0000000000000000000000000000000000000000..6b1dea67f2dc12fd124f0094bde1737fd026fbf3 --- /dev/null +++ b/dev-support/style-check/lint-java.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" +mvn --no-transfer-progress install -DskipTests -pl "!org.apache.submarine:submarine-cloud" + +ERRORS=$(mvn checkstyle:check | grep ERROR) + +if test ! -z "$ERRORS"; then + echo -e "Checkstyle checks failed at following occurrences:\n$ERRORS" + find . -name checkstyle-output.xml | xargs cat | grep -v checkstyle | grep -v " +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of leveldbjni +(https://github.com/fusesource/leveldbjni), which is available under the +following license: + +Copyright (c) 2011 FuseSource Corp. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of FuseSource Corp. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: + +Copyright (c) 2012 The FreeBSD Foundation +All rights reserved. + +This software was developed by Pawel Jakub Dawidek under sponsorship from +the FreeBSD Foundation. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +============= + +The binary distribution of this product bundles binaries of leveldb +(http://code.google.com/p/leveldb/), which is available under the following +license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles binaries of snappy +(http://code.google.com/p/snappy/), which is available under the following +license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/ +-------------------------------------------------------------------------------- +Copyright (C) 2008-2016, SpryMedia Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2010 Aleksander Williams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js +-------------------------------------------------------------------------------- + +Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +The binary distribution of this product bundles these dependencies under the +following license: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.3.7 +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css +hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css +bootstrap v3.3.6 +broccoli-asset-rev v2.4.2 +broccoli-funnel v1.0.1 +datatables v1.10.8 +em-helpers v0.5.13 +em-table v0.1.6 +ember v2.2.0 +ember-array-contains-helper v1.0.2 +ember-bootstrap v0.5.1 +ember-cli v1.13.13 +ember-cli-app-version v1.0.0 +ember-cli-babel v5.1.6 +ember-cli-content-security-policy v0.4.0 +ember-cli-dependency-checker v1.2.0 +ember-cli-htmlbars v1.0.2 +ember-cli-htmlbars-inline-precompile v0.3.1 +ember-cli-ic-ajax v0.2.1 +ember-cli-inject-live-reload v1.4.0 +ember-cli-jquery-ui v0.0.20 +ember-cli-qunit v1.2.1 +ember-cli-release v0.2.8 +ember-cli-shims v0.0.6 +ember-cli-sri v1.2.1 +ember-cli-test-loader v0.2.1 +ember-cli-uglify v1.2.0 +ember-d3 v0.1.0 +ember-data v2.1.0 +ember-disable-proxy-controllers v1.0.1 +ember-export-application-global v1.0.5 +ember-load-initializers v0.1.7 +ember-qunit v0.4.16 +ember-qunit-notifications v0.1.0 +ember-resolver v2.0.3 +ember-spin-spinner v0.2.3 +ember-truth-helpers v1.2.0 +jquery v2.1.4 +jquery-ui v1.11.4 +loader.js v3.3.0 +momentjs v2.10.6 +qunit v1.19.0 +select2 v4.0.0 +snippet-ss v1.11.0 +spin.js v2.3.2 +Azure Data Lake Store - Java client SDK 2.0.11 +JCodings 1.0.8 +Joni 2.1.2 +Mockito 1.8.5 +JUL to SLF4J bridge 1.7.25 +SLF4J API Module 1.7.25 +SLF4J LOG4J-12 Binding 1.7.25 +-------------------------------------------------------------------------------- + +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +For: +./hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js files) +-------------------------------------------------------------------------------- +Copyright (c) 2011-2014 Novus Partners, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this +file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + express or implied. See the License for the specific language governing permissions and + limitations under the License. + + + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js +-------------------------------------------------------------------------------- +The MIT License (MIT) +Copyright (c) 2014 Konstantin Skipor + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js +-------------------------------------------------------------------------------- +The MIT License + +Copyright (c) 2010-2017 Google, Inc. http://angularjs.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + + + +For: +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.3.1.min.js +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery +Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 +Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2003-2017 Optimatika + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +For: +oj! Algorithms - version 43.0 +-------------------------------------------------------------------------------- + +Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + +==== + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +==== + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. + +For: +hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js +-------------------------------------------------------------------------------- + +Copyright (c) 2014 Ivan Bozhanov + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js +-------------------------------------------------------------------------------- + +D3 is available under a 3-clause BSD license. For details, see: +hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE + +The binary distribution of this product bundles these dependencies under the +following license: +HSQLDB Database 2.3.4 +-------------------------------------------------------------------------------- +(HSQL License) +"COPYRIGHTS AND LICENSES (based on BSD License) + +For work developed by the HSQL Development Group: + +Copyright (c) 2001-2016, The HSQL Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the HSQL Development Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +For work originally developed by the Hypersonic SQL Group: + +Copyright (c) 1995-2000 by the Hypersonic SQL Group. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +Neither the name of the Hypersonic SQL Group nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This software consists of voluntary contributions made by many individuals on behalf of the +Hypersonic SQL Group." + +The binary distribution of this product bundles these dependencies under the +following license: +Java Servlet API 3.1.0 +servlet-api 2.5 +jsp-api 2.1 +jsr311-api 1.1.1 +Glassfish Jasper 6.1.14 +Servlet Specification 2.5 API 6.1.14 +-------------------------------------------------------------------------------- +(CDDL 1.0) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 +1. Definitions.  + +1.1. Contributor means each individual or entity +that creates or contributes to the creation of +Modifications.  + +1.2. Contributor Version means the combination of the +Original Software, prior Modifications used by a Contributor (if any), and the +Modifications made by that particular Contributor.  + +1.3. Covered +Software means (a) the Original Software, or (b) Modifications, or (c) the +combination of files containing Original Software with files containing +Modifications, in each case including portions +thereof.  + +1.4. Executable means the Covered Software in any form other +than Source Code.  + +1.5. Initial Developer means the individual or entity +that first makes Original Software available under this +License.  + +1.6. Larger Work means a work which combines Covered Software or +portions thereof with code not governed by the terms of this +License.  + +1.7. License means this document.  + +1.8. Licensable means +having the right to grant, to the maximum extent possible, whether at the time +of the initial grant or subsequently acquired, any and all of the rights +conveyed herein.  + +1.9. Modifications means the Source Code and Executable +form of any of the following: +A. Any file that results from an addition to, +deletion from or modification of the contents of a file containing Original +Software or previous Modifications; +B. Any new file that contains any part of the Original Software +or previous Modification; or +C. Any new file that is contributed or otherwise made available +under the terms of this License.  + +1.10. Original Software means the Source Code and Executable form of +computer software code that is originally released under this License.  + +1.11. Patent Claims means any patent claim(s), now owned or +hereafter acquired, including without limitation, method, process, and apparatus +claims, in any patent Licensable by grantor.  + +1.12. Source Code means (a) the common form of computer software code in which +modifications are made and (b) associated documentation included in or +with such code.  + +1.13. You (or Your) means an individual or a legal entity exercising rights +under, and complying with all of the terms of, this License. For legal entities, +You includes any entity which controls, is controlled by, or is under common control +with You. For purposes of this definition, control means (a) the power, direct +or indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (b) ownership of more than fifty percent (50%) of the +outstanding shares or beneficial ownership of such entity.  + +2. License Grants. + +2.1. The Initial Developer Grant. Conditioned upon Your compliance +with Section 3.1 below and subject to third party intellectual property claims, +the Initial Developer hereby grants You a world-wide, royalty-free, +non-exclusive license:  + +(a) under intellectual property rights (other than +patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, +display, perform, sublicense and distribute the Original Software (or portions +thereof), with or without Modifications, and/or as part of a Larger Work; +and  + +(b) under Patent Claims infringed by the making, using or selling of +Original Software, to make, have made, use, practice, sell, and offer for sale, +and/or otherwise dispose of the Original Software (or portions +thereof); + +(c) The licenses granted in Sections 2.1(a) and (b) are +effective on the date Initial Developer first distributes or otherwise makes the +Original Software available to a third party under the terms of this +License; + +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or +(ii) the combination of the Original Software with other software or +devices.  + +2.2. Contributor Grant. Conditioned upon Your compliance with +Section 3.1 below and subject to third party intellectual property claims, each +Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license:  + +(a) under intellectual property rights (other than patent or +trademark) Licensable by Contributor to use, reproduce, modify, display, +perform, sublicense and distribute the Modifications created by such Contributor +(or portions thereof), either on an unmodified basis, with other Modifications, +as Covered Software and/or as part of a Larger Work; and  + +(b) under Patent +Claims infringed by the making, using, or selling of Modifications made by that +Contributor either alone and/or in combination with its Contributor Version (or +portions of such combination), to make, use, sell, offer for sale, have made, +and/or otherwise dispose of: (1) Modifications made by that Contributor (or +portions thereof); and (2) the combination of Modifications made by that +Contributor with its Contributor Version (or portions of such +combination).  + +(c) The licenses granted in Sections 2.2(a) and 2.2(b) are +effective on the date Contributor first distributes or otherwise makes the +Modifications available to a third party. + +(d) Notwithstanding Section 2.2(b) +above, no patent license is granted: (1) for any code that Contributor has +deleted from the Contributor Version; (2) for infringements caused by: +(i) third party modifications of Contributor Version, or (ii) the combination +of Modifications made by that Contributor with other software (except as part of +the Contributor Version) or other devices; or (3) under Patent Claims infringed +by Covered Software in the absence of Modifications made by that +Contributor.  + +3. Distribution Obligations.  + +3.1. Availability of Source +Code. Any Covered Software that You distribute or otherwise make available in +Executable form must also be made available in Source Code form and that Source +Code form must be distributed only under the terms of this License. You must +include a copy of this License with every copy of the Source Code form of the +Covered Software You distribute or otherwise make available. You must inform +recipients of any such Covered Software in Executable form as to how they can +obtain such Covered Software in Source Code form in a reasonable manner on or +through a medium customarily used for software exchange.  + +3.2. +Modifications. The Modifications that You create or to which You contribute are +governed by the terms of this License. You represent that You believe Your +Modifications are Your original creation(s) and/or You have sufficient rights to +grant the rights conveyed by this License.  + +3.3. Required Notices. You must +include a notice in each of Your Modifications that identifies You as the +Contributor of the Modification. You may not remove or alter any copyright, +patent or trademark notices contained within the Covered Software, or any +notices of licensing or any descriptive text giving attribution to any +Contributor or the Initial Developer.  + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code +form that alters or restricts the applicable version of this License or the +recipients rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. + +3.5. Distribution of Executable Versions. You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipients rights in the Source Code form from the rights set forth in this +License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer.  + +3.6. Larger Works. You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software.  + +4. Versions of the License.  + +4.1. +New Versions. Sun Microsystems, Inc. is the initial license steward and may +publish revised and/or new versions of this License from time to time. Each +version will be given a distinguishing version number. Except as provided in +Section 4.3, no one other than the license steward has the right to modify this +License.  + +4.2. Effect of New Versions. You may always continue to use, +distribute or otherwise make the Covered Software available under the terms of +the version of the License under which You originally received the Covered +Software. If the Initial Developer includes a notice in the Original Software +prohibiting it from being distributed or otherwise made available under any +subsequent version of the License, You must distribute and make the Covered +Software available under the terms of the version of the License under which You +originally received the Covered Software. Otherwise, You may also choose to use, +distribute or otherwise make the Covered Software available under the terms of +any subsequent version of the License published by the license +steward.  + +4.3. Modified Versions. When You are an Initial Developer and You +want to create a new license for Your Original Software, You may create and use +a modified version of this License if You: (a) rename the license and remove +any references to the name of the license steward (except to note that the +license differs from this License); and (b) otherwise make it clear that the +license contains terms which differ from this License.  + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, +MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY +COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER +OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR +CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS +LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER +THIS DISCLAIMER.  + +6. TERMINATION.  + +6.1. This License and the rights +granted hereunder will terminate automatically if You fail to comply with terms +herein and fail to cure such breach within 30 days of becoming aware of the +breach. Provisions which, by their nature, must remain in effect beyond the +termination of this License shall survive.  + +6.2. If You assert a patent +infringement claim (excluding declaratory judgment actions) against Initial +Developer or a Contributor (the Initial Developer or Contributor against whom +You assert such claim is referred to as Participant) alleging that the +Participant Software (meaning the Contributor Version where the Participant is a +Contributor or the Original Software where the Participant is the Initial +Developer) directly or indirectly infringes any patent, then any and all rights +granted directly or indirectly to You by such Participant, the Initial Developer +(if the Initial Developer is not the Participant) and all Contributors under +Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from +Participant terminate prospectively and automatically at the expiration of such +60 day notice period, unless if within such 60 day period You withdraw Your +claim with respect to the Participant Software against such Participant either +unilaterally or pursuant to a written agreement with Participant.  + +6.3. In +the event of termination under Sections 6.1 or 6.2 above, all end user licenses +that have been validly granted by You or any distributor hereunder prior to +termination (excluding licenses granted to You by any distributor) shall survive +termination.  + +7. LIMITATION OF LIABILITY. +UNDER NO CIRCUMSTANCES AND UNDER +NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, +SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF +COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY +PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY +CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF +GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER +COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE +POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO +LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO +THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT +ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO +THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a commercial item, as that term is defined in +48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as +that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer +software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. +1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through +227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software +with only those rights set forth herein. This U.S. Government Rights clause is +in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision +that addresses Government rights in computer software under this +License.  + +9. MISCELLANEOUS. +This License represents the complete agreement +concerning subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdictions conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software.  + +10. RESPONSIBILITY FOR CLAIMS. +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability.  + +The binary distribution of this product bundles these dependencies under the +following license: +jersey-client 1.19 +jersey-core 1.19 +jersey-grizzly2 1.19 +jersey-grizzly2-servlet 1.19 +jersey-json 1.19 +jersey-server 1.19 +jersey-servlet 1.19 +jersey-guice 1.19 +Jersey Test Framework - Grizzly 2 Module 1.19 +JAXB RI 2.2.3 +Java Architecture for XML Binding 2.2.11 +grizzly-framework 2.2.21 +grizzly-http 2.2.21 +grizzly-http-server 2.2.21 +grizzly-http-servlet 2.2.21 +grizzly-rcm 2.2.21 +JavaBeans Activation Framework 1.2.0 +-------------------------------------------------------------------------------- +(CDDL 1.1) +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or +contributes to the creation of Modifications. +1.2. “Contributor Version” means the combination of the Original Software, +prior Modifications used by a Contributor (if any), and the Modifications made +by that particular Contributor. +1.3. “Covered Software” means (a) the Original Software, or (b) +Modifications, or (c) the combination of files containing Original Software with +files containing Modifications, in each case including portions thereof. +1.4. “Executable” means the Covered Software in any form other than Source +Code. +1.5. “Initial Developer” means the individual or entity that first makes +Original Software available under this License. +1.6. “Larger Work” means a work which combines Covered Software or portions +thereof with code not governed by the terms of this License. +1.7. “License” means this document. +1.8. “Licensable” means having the right to grant, to the maximum extent +possible, whether at the time of the initial grant or subsequently acquired, any +and all of the rights conveyed herein. +1.9. “Modifications” means the Source Code and Executable form of any of the +following: +A. Any file that results from an addition to, deletion from or modification of +the contents of a file containing Original Software or previous Modifications; +B. Any new file that contains any part of the Original Software or previous +Modification; or +C. Any new file that is contributed or otherwise made available under the terms +of this License. +1.10. “Original Software” means the Source Code and Executable form of +computer software code that is originally released under this License. +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter +acquired, including without limitation, method, process, and apparatus claims, +in any patent Licensable by grantor. +1.12. “Source Code” means (a) the common form of computer software code in +which modifications are made and (b) associated documentation included in or +with such code. +1.13. “You” (or “Your”) means an individual or a legal entity exercising +rights under, and complying with all of the terms of, this License. For legal +entities, “You” includes any entity which controls, is controlled by, or is +under common control with You. For purposes of this definition, “control” +means (a) the power, direct or indirect, to cause the direction or management of +such entity, whether by contract or otherwise, or (b) ownership of more than +fifty percent (50%) of the outstanding shares or beneficial ownership of such +entity. + +2. License Grants. + +2.1. The Initial Developer Grant. + +Conditioned upon Your compliance with Section 3.1 below and subject to +third party intellectual property claims, the Initial Developer hereby grants +You a world-wide, royalty-free, non-exclusive license: +(a) under intellectual +property rights (other than patent or trademark) Licensable by Initial +Developer, to use, reproduce, modify, display, perform, sublicense and +distribute the Original Software (or portions thereof), with or without +Modifications, and/or as part of a Larger Work; and +(b) under Patent Claims +infringed by the making, using or selling of Original Software, to make, have +made, use, practice, sell, and offer for sale, and/or otherwise dispose of the +Original Software (or portions thereof). +(c) The licenses granted in Sections +2.1(a) and (b) are effective on the date Initial Developer first distributes or +otherwise makes the Original Software available to a third party under the terms +of this License. +(d) Notwithstanding Section 2.1(b) above, no patent license is +granted: (1) for code that You delete from the Original Software, or (2) for +infringements caused by: (i) the modification of the Original Software, or (ii) +the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. + +Conditioned upon Your compliance with Section 3.1 below and +subject to third party intellectual property claims, each Contributor hereby +grants You a world-wide, royalty-free, non-exclusive license: +(a) under +intellectual property rights (other than patent or trademark) Licensable by +Contributor to use, reproduce, modify, display, perform, sublicense and +distribute the Modifications created by such Contributor (or portions thereof), +either on an unmodified basis, with other Modifications, as Covered Software +and/or as part of a Larger Work; and +(b) under Patent Claims infringed by the +making, using, or selling of Modifications made by that Contributor either alone +and/or in combination with its Contributor Version (or portions of such +combination), to make, use, sell, offer for sale, have made, and/or otherwise +dispose of: (1) Modifications made by that Contributor (or portions thereof); +and (2) the combination of Modifications made by that Contributor with its +Contributor Version (or portions of such combination). +(c) The licenses granted +in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first +distributes or otherwise makes the Modifications available to a third +party. +(d) Notwithstanding Section 2.2(b) above, no patent license is granted: +(1) for any code that Contributor has deleted from the Contributor Version; (2) +for infringements caused by: (i) third party modifications of Contributor +Version, or (ii) the combination of Modifications made by that Contributor with +other software (except as part of the Contributor Version) or other devices; or +(3) under Patent Claims infringed by Covered Software in the absence of +Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or +otherwise make available in Executable form must also be made available in +Source Code form and that Source Code form must be distributed only under the +terms of this License. You must include a copy of this License with every copy +of the Source Code form of the Covered Software You distribute or otherwise make +available. You must inform recipients of any such Covered Software in Executable +form as to how they can obtain such Covered Software in Source Code form in a +reasonable manner on or through a medium customarily used for software +exchange. +3.2. Modifications. +The Modifications that You create or to which +You contribute are governed by the terms of this License. You represent that You +believe Your Modifications are Your original creation(s) and/or You have +sufficient rights to grant the rights conveyed by this License. +3.3. Required Notices. +You must include a notice in each of Your Modifications that +identifies You as the Contributor of the Modification. You may not remove or +alter any copyright, patent or trademark notices contained within the Covered +Software, or any notices of licensing or any descriptive text giving attribution +to any Contributor or the Initial Developer. +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source +Code form that alters or restricts the applicable version of this License or the +recipients' rights hereunder. You may choose to offer, and to charge a fee for, +warranty, support, indemnity or liability obligations to one or more recipients +of Covered Software. However, you may do so only on Your own behalf, and not on +behalf of the Initial Developer or any Contributor. You must make it absolutely +clear that any such warranty, support, indemnity or liability obligation is +offered by You alone, and You hereby agree to indemnify the Initial Developer +and every Contributor for any liability incurred by the Initial Developer or +such Contributor as a result of warranty, support, indemnity or liability terms +You offer. +3.5. Distribution of Executable Versions. +You may distribute the +Executable form of the Covered Software under the terms of this License or under +the terms of a license of Your choice, which may contain terms different from +this License, provided that You are in compliance with the terms of this License +and that the license for the Executable form does not attempt to limit or alter +the recipient's rights in the Source Code form from the rights set forth in +this License. If You distribute the Covered Software in Executable form under a +different license, You must make it absolutely clear that any terms which differ +from this License are offered by You alone, not by the Initial Developer or +Contributor. You hereby agree to indemnify the Initial Developer and every +Contributor for any liability incurred by the Initial Developer or such +Contributor as a result of any such terms You offer. +3.6. Larger Works. +You +may create a Larger Work by combining Covered Software with other code not +governed by the terms of this License and distribute the Larger Work as a single +product. In such a case, You must make sure the requirements of this License are +fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Oracle is the initial license steward and may publish revised and/or +new versions of this License from time to time. Each version will be given a +distinguishing version number. Except as provided in Section 4.3, no one other +than the license steward has the right to modify this License. +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the +Covered Software available under the terms of the version of the License under +which You originally received the Covered Software. If the Initial Developer +includes a notice in the Original Software prohibiting it from being distributed +or otherwise made available under any subsequent version of the License, You +must distribute and make the Covered Software available under the terms of the +version of the License under which You originally received the Covered Software. +Otherwise, You may also choose to use, distribute or otherwise make the Covered +Software available under the terms of any subsequent version of the License +published by the license steward. +4.3. Modified Versions. +When You are an +Initial Developer and You want to create a new license for Your Original +Software, You may create and use a modified version of this License if You: (a) +rename the license and remove any references to the name of the license steward +(except to note that the license differs from this License); and (b) otherwise +make it clear that the license contains terms which differ from this +License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS +LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE +IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR +NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED +SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY +RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE +COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF +WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED +SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will +terminate automatically if You fail to comply with terms herein and fail to cure +such breach within 30 days of becoming aware of the breach. Provisions which, by +their nature, must remain in effect beyond the termination of this License shall +survive. +6.2. If You assert a patent infringement claim (excluding declaratory +judgment actions) against Initial Developer or a Contributor (the Initial +Developer or Contributor against whom You assert such claim is referred to as +“Participant”) alleging that the Participant Software (meaning the +Contributor Version where the Participant is a Contributor or the Original +Software where the Participant is the Initial Developer) directly or indirectly +infringes any patent, then any and all rights granted directly or indirectly to +You by such Participant, the Initial Developer (if the Initial Developer is not +the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this +License shall, upon 60 days notice from Participant terminate prospectively and +automatically at the expiration of such 60 day notice period, unless if within +such 60 day period You withdraw Your claim with respect to the Participant +Software against such Participant either unilaterally or pursuant to a written +agreement with Participant. +6.3. If You assert a patent infringement claim +against Participant alleging that the Participant Software directly or +indirectly infringes any patent where such claim is resolved (such as by license +or settlement) prior to the initiation of patent infringement litigation, then +the reasonable value of the licenses granted by such Participant under Sections +2.1 or 2.2 shall be taken into account in determining the amount or value of any +payment or license. +6.4. In the event of termination under Sections 6.1 or 6.2 +above, all end user licenses that have been validly granted by You or any +distributor hereunder prior to termination (excluding licenses granted to You by +any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT +(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL +DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY +SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, +WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER +FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN +IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS +LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL +INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW +PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR +LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND +LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered +Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 +(Oct. 1995), consisting of “commercial computer software” (as that term is +defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software +documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). +Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 +(June 1995), all U.S. Government End Users acquire Covered Software with only +those rights set forth herein. This U.S. Government Rights clause is in lieu of, +and supersedes, any other FAR, DFAR, or other clause or provision that addresses +Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning +subject matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent necessary to +make it enforceable. This License shall be governed by the law of the +jurisdiction specified in a notice contained within the Original Software +(except to the extent applicable law, if any, provides otherwise), excluding +such jurisdiction's conflict-of-law provisions. Any litigation relating to this +License shall be subject to the jurisdiction of the courts located in the +jurisdiction and venue specified in a notice contained within the Original +Software, with the losing party responsible for costs, including, without +limitation, court costs and reasonable attorneys' fees and expenses. The +application of the United Nations Convention on Contracts for the International +Sale of Goods is expressly excluded. Any law or regulation which provides that +the language of a contract shall be construed against the drafter shall not +apply to this License. You agree that You alone are responsible for compliance +with the United States export administration regulations (and the export control +laws and regulation of any other countries) when You use, distribute or +otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is +responsible for claims and damages arising, directly or indirectly, out of its +utilization of rights under this License and You agree to work with Initial +Developer and Contributors to distribute such responsibility on an equitable +basis. Nothing herein is intended or shall be deemed to constitute any admission +of liability. + +The binary distribution of this product bundles these dependencies under the +following license: +Protocol Buffer Java API 2.5.0 +-------------------------------------------------------------------------------- +This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + +Copyright 2014, Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +For: +XML Commons External Components XML APIs 1.3.04 +-------------------------------------------------------------------------------- +By obtaining, using and/or copying this work, you (the licensee) agree that you +have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, +with or without modification, for any purpose and without fee or royalty is +hereby granted, provided that you include the following on ALL copies of the +software and documentation or portions thereof, including modifications: +- The full text of this NOTICE in a location viewable to users of the +redistributed or derivative work. +- Any pre-existing intellectual property disclaimers, notices, or terms and +conditions. If none exist, the W3C Software Short Notice should be included +(hypertext is preferred, text is permitted) within the body of any redistributed +or derivative code. +- Notice of any changes or modifications to the files, including the date changes +were made. (We recommend you provide URIs to the location from which the code is +derived.) + +The binary distribution of this product bundles these dependencies under the +following license: +JUnit 4.11 +Eclipse JDT Core 3.1.1 +-------------------------------------------------------------------------------- +(EPL v1.0) +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation +distributed under this Agreement, and +b) in the case of each subsequent Contributor: +i) changes to the Program, and +ii) additions to the Program; +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' from a +Contributor if it was added to the Program by such Contributor itself or anyone +acting on such Contributor's behalf. Contributions do not include additions to +the Program which: (i) are separate modules of software distributed in +conjunction with the Program under their own license agreement, and (ii) are not +derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or when +combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free copyright license to +reproduce, prepare derivative works of, publicly display, publicly perform, +distribute and sublicense the Contribution of such Contributor, if any, and such +derivative works, in source code and object code form. +b) Subject to the terms of this Agreement, each Contributor hereby grants +Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed +Patents to make, use, sell, offer to sell, import and otherwise transfer the +Contribution of such Contributor, if any, in source code and object code form. +This patent license shall apply to the combination of the Contribution and the +Program if, at the time the Contribution is added by the Contributor, such +addition of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other combinations +which include the Contribution. No hardware per se is licensed hereunder. +c) Recipient understands that although each Contributor grants the licenses to +its Contributions set forth herein, no assurances are provided by any +Contributor that the Program does not infringe the patent or other intellectual +property rights of any other entity. Each Contributor disclaims any liability to +Recipient for claims brought by any other entity based on infringement of +intellectual property rights or otherwise. As a condition to exercising the +rights and licenses granted hereunder, each Recipient hereby assumes sole +responsibility to secure any other intellectual property rights needed, if any. +For example, if a third party patent license is required to allow Recipient to +distribute the Program, it is Recipient's responsibility to acquire that license +before distributing the Program. +d) Each Contributor represents that to its knowledge it has sufficient copyright +rights in its Contribution, if any, to grant the copyright license set forth in +this Agreement. +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its +own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and +b) its license agreement: +i) effectively disclaims on behalf of all Contributors all warranties and +conditions, express and implied, including warranties or conditions of title and +non-infringement, and implied warranties or conditions of merchantability and +fitness for a particular purpose; +ii) effectively excludes on behalf of all Contributors all liability for +damages, including direct, indirect, special, incidental and consequential +damages, such as lost profits; +iii) states that any provisions which differ from this Agreement are offered by +that Contributor alone and not by any other party; and +iv) states that source code for the Program is available from such Contributor, +and informs licensees how to obtain it in a reasonable manner on or through a +medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the +Program. + +Each Contributor must identify itself as the originator of its Contribution, if +any, in a manner that reasonably allows subsequent Recipients to identify the +originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, if +a Contributor includes the Program in a commercial product offering, such +Contributor ("Commercial Contributor") hereby agrees to defend and indemnify +every other Contributor ("Indemnified Contributor") against any losses, damages +and costs (collectively "Losses") arising from claims, lawsuits and other legal +actions brought by a third party against the Indemnified Contributor to the +extent caused by the acts or omissions of such Commercial Contributor in +connection with its distribution of the Program in a commercial product +offering. The obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property infringement. In order +to qualify, an Indemnified Contributor must: a) promptly notify the Commercial +Contributor in writing of such claim, and b) allow the Commercial Contributor to +control, and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may participate in +any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. If that +Commercial Contributor then makes performance claims, or offers warranties +related to Product X, those performance claims and warranties are such +Commercial Contributor's responsibility alone. Under this section, the +Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a court +requires any other Contributor to pay any damages as a result, the Commercial +Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each +Recipient is solely responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its exercise of +rights under this Agreement , including but not limited to the risks and costs +of program errors, compliance with applicable laws, damage to or loss of data, +programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS +GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this Agreement, and without further action by the parties hereto, such +provision shall be reformed to the minimum extent necessary to make such +provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted under +Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and does +not cure such failure in a reasonable period of time after becoming aware of +such noncompliance. If all Recipient's rights under this Agreement terminate, +Recipient agrees to cease use and distribution of the Program as soon as +reasonably practicable. However, Recipient's obligations under this Agreement +and any licenses granted by Recipient relating to the Program shall continue and +survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in +order to avoid inconsistency the Agreement is copyrighted and may only be +modified in the following manner. The Agreement Steward reserves the right to +publish new versions (including revisions) of this Agreement from time to time. +No one other than the Agreement Steward has the right to modify this Agreement. +The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation +may assign the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) may always +be distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to distribute the Program (including its Contributions) +under the new version. Except as expressly stated in Sections 2(a) and 2(b) +above, Recipient receives no rights or licenses to the intellectual property of +any Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted under +this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to this +Agreement will bring a legal action under this Agreement more than one year +after the cause of action arose. Each party waives its rights to a jury trial in +any resulting litigation. + +The binary distribution of this product bundles these dependencies under the +following license: +JSch 0.1.54 +ParaNamer Core 2.3 +JLine 0.9.94 +leveldbjni-all 1.8 +Hamcrest Core 1.3 +ASM Core 5.0.4 +ASM Commons 5.0.2 +ASM Tree 5.0.2 +-------------------------------------------------------------------------------- +(3-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles these dependencies under the +following license: +FindBugs-jsr305 3.0.0 +dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. +-------------------------------------------------------------------------------- +(2-clause BSD) +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. + +The binary distribution of this product bundles these dependencies under the +following license: +"Java Concurrency in Practice" book annotations 1.0 +-------------------------------------------------------------------------------- +(CCAL v2.5) +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS +PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR +OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS +LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE +BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED +HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + +"Collective Work" means a work, such as a periodical issue, anthology or +encyclopedia, in which the Work in its entirety in unmodified form, along with a +number of other contributions, constituting separate and independent works in +themselves, are assembled into a collective whole. A work that constitutes a +Collective Work will not be considered a Derivative Work (as defined below) for +the purposes of this License. +"Derivative Work" means a work based upon the Work or upon the Work and other +pre-existing works, such as a translation, musical arrangement, dramatization, +fictionalization, motion picture version, sound recording, art reproduction, +abridgment, condensation, or any other form in which the Work may be recast, +transformed, or adapted, except that a work that constitutes a Collective Work +will not be considered a Derivative Work for the purpose of this License. For +the avoidance of doubt, where the Work is a musical composition or sound +recording, the synchronization of the Work in timed-relation with a moving image +("synching") will be considered a Derivative Work for the purpose of this +License. +"Licensor" means the individual or entity that offers the Work under the terms +of this License. +"Original Author" means the individual or entity who created the Work. +"Work" means the copyrightable work of authorship offered under the terms of +this License. +"You" means an individual or entity exercising rights under this License who has +not previously violated the terms of this License with respect to the Work, or +who has received express permission from the Licensor to exercise rights under +this License despite a previous violation. +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or +restrict any rights arising from fair use, first sale or other limitations on +the exclusive rights of the copyright owner under copyright law or other +applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor +hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the +duration of the applicable copyright) license to exercise the rights in the Work +as stated below: + +to reproduce the Work, to incorporate the Work into one or more Collective +Works, and to reproduce the Work as incorporated in the Collective Works; +to create and reproduce Derivative Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission the Work including as +incorporated in Collective Works; +to distribute copies or phonorecords of, display publicly, perform publicly, and +perform publicly by means of a digital audio transmission Derivative Works. +For the avoidance of doubt, where the work is a musical composition: + +Performance Royalties Under Blanket Licenses. Licensor waives the exclusive +right to collect, whether individually or via a performance rights society (e.g. +ASCAP, BMI, SESAC), royalties for the public performance or public digital +performance (e.g. webcast) of the Work. +Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right +to collect, whether individually or via a music rights agency or designated +agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the +Work ("cover version") and distribute, subject to the compulsory license created +by 17 USC Section 115 of the US Copyright Act (or the equivalent in other +jurisdictions). +Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the +Work is a sound recording, Licensor waives the exclusive right to collect, +whether individually or via a performance-rights society (e.g. SoundExchange), +royalties for the public digital performance (e.g. webcast) of the Work, subject +to the compulsory license created by 17 USC Section 114 of the US Copyright Act +(or the equivalent in other jurisdictions). +The above rights may be exercised in all media and formats whether now known or +hereafter devised. The above rights include the right to make such modifications +as are technically necessary to exercise the rights in other media and formats. +All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject +to and limited by the following restrictions: + +You may distribute, publicly display, publicly perform, or publicly digitally +perform the Work only under the terms of this License, and You must include a +copy of, or the Uniform Resource Identifier for, this License with every copy or +phonorecord of the Work You distribute, publicly display, publicly perform, or +publicly digitally perform. You may not offer or impose any terms on the Work +that alter or restrict the terms of this License or the recipients' exercise of +the rights granted hereunder. You may not sublicense the Work. You must keep +intact all notices that refer to this License and to the disclaimer of +warranties. You may not distribute, publicly display, publicly perform, or +publicly digitally perform the Work with any technological measures that control +access or use of the Work in a manner inconsistent with the terms of this +License Agreement. The above applies to the Work as incorporated in a Collective +Work, but this does not require the Collective Work apart from the Work itself +to be made subject to the terms of this License. If You create a Collective +Work, upon notice from any Licensor You must, to the extent practicable, remove +from the Collective Work any credit as required by clause 4(b), as requested. If +You create a Derivative Work, upon notice from any Licensor You must, to the +extent practicable, remove from the Derivative Work any credit as required by +clause 4(b), as requested. +If you distribute, publicly display, publicly perform, or publicly digitally +perform the Work or any Derivative Works or Collective Works, You must keep +intact all copyright notices for the Work and provide, reasonable to the medium +or means You are utilizing: (i) the name of the Original Author (or pseudonym, +if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor +designate another party or parties (e.g. a sponsor institute, publishing entity, +journal) for attribution in Licensor's copyright notice, terms of service or by +other reasonable means, the name of such party or parties; the title of the Work +if supplied; to the extent reasonably practicable, the Uniform Resource +Identifier, if any, that Licensor specifies to be associated with the Work, +unless such URI does not refer to the copyright notice or licensing information +for the Work; and in the case of a Derivative Work, a credit identifying the use +of the Work in the Derivative Work (e.g., "French translation of the Work by +Original Author," or "Screenplay based on original Work by Original Author"). +Such credit may be implemented in any reasonable manner; provided, however, that +in the case of a Derivative Work or Collective Work, at a minimum such credit +will appear where any other comparable authorship credit appears and in a manner +at least as prominent as such other comparable authorship credit. +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS +THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING +THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT +LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR +PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, +OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME +JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH +EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN +NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, +INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS +LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. Termination + +This License and the rights granted hereunder will terminate automatically upon +any breach by You of the terms of this License. Individuals or entities who have +received Derivative Works or Collective Works from You under this License, +however, will not have their licenses terminated provided such individuals or +entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, +and 8 will survive any termination of this License. +Subject to the above terms and conditions, the license granted here is perpetual +(for the duration of the applicable copyright in the Work). Notwithstanding the +above, Licensor reserves the right to release the Work under different license +terms or to stop distributing the Work at any time; provided, however that any +such election will not serve to withdraw this License (or any other license that +has been, or is required to be, granted under the terms of this License), and +this License will continue in full force and effect unless terminated as stated +above. +8. Miscellaneous + +Each time You distribute or publicly digitally perform the Work or a Collective +Work, the Licensor offers to the recipient a license to the Work on the same +terms and conditions as the license granted to You under this License. +Each time You distribute or publicly digitally perform a Derivative Work, +Licensor offers to the recipient a license to the original Work on the same +terms and conditions as the license granted to You under this License. +If any provision of this License is invalid or unenforceable under applicable +law, it shall not affect the validity or enforceability of the remainder of the +terms of this License, and without further action by the parties to this +agreement, such provision shall be reformed to the minimum extent necessary to +make such provision valid and enforceable. +No term or provision of this License shall be deemed waived and no breach +consented to unless such waiver or consent shall be in writing and signed by the +party to be charged with such waiver or consent. +This License constitutes the entire agreement between the parties with respect +to the Work licensed here. There are no understandings, agreements or +representations with respect to the Work not specified here. Licensor shall not +be bound by any additional provisions that may appear in any communication from +You. This License may not be modified without the mutual written agreement of +the Licensor and You. + +The binary distribution of this product bundles these dependencies under the +following license: +jamon-runtime 2.4.1 +-------------------------------------------------------------------------------- +(MPL 2.0) + Mozilla Public License + Version 2.0 + +1. Definitions + +1.1. “Contributor” +means each individual or legal entity that creates, contributes to the creation +of, or owns Covered Software. + +1.2. “Contributor Version” +means the combination of the Contributions of others (if any) used by a +Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” +means Covered Software of a particular Contributor. + +1.4. “Covered Software” +means Source Code Form to which the initial Contributor has attached the notice +in Exhibit A, the Executable Form of such Source Code Form, and Modifications of +such Source Code Form, in each case including portions thereof. + +1.5. “Incompatible With Secondary Licenses” +means + +that the initial Contributor has attached the notice described in Exhibit B to +the Covered Software; or + +that the Covered Software was made available under the terms of version 1.1 or +earlier of the License, but not also under the terms of a Secondary License. + +1.6. “Executable Form” +means any form of the work other than Source Code Form. + +1.7. “Larger Work” +means a work that combines Covered Software with other material, in a separate +file or files, that is not Covered Software. + +1.8. “License” +means this document. + +1.9. “Licensable” +means having the right to grant, to the maximum extent possible, whether at the +time of the initial grant or subsequently, any and all of the rights conveyed by +this License. + +1.10. “Modifications” +means any of the following: + +any file in Source Code Form that results from an addition to, deletion from, or +modification of the contents of Covered Software; or + +any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor +means any patent claim(s), including without limitation, method, process, and +apparatus claims, in any patent Licensable by such Contributor that would be +infringed, but for the grant of the License, by the making, using, selling, +offering for sale, having made, import, or transfer of either its Contributions +or its Contributor Version. + +1.12. “Secondary License” +means either the GNU General Public License, Version 2.0, the GNU Lesser General +Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, +or any later versions of those licenses. + +1.13. “Source Code Form” +means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) +means an individual or a legal entity exercising rights under this License. For +legal entities, “You” includes any entity that controls, is controlled by, +or is under common control with You. For purposes of this definition, +“control” means (a) the power, direct or indirect, to cause the direction or +management of such entity, whether by contract or otherwise, or (b) ownership of +more than fifty percent (50%) of the outstanding shares or beneficial ownership +of such entity. + +2. License Grants and Conditions + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive +license: + +under intellectual property rights (other than patent or trademark) Licensable +by such Contributor to use, reproduce, make available, modify, display, perform, +distribute, and otherwise exploit its Contributions, either on an unmodified +basis, with Modifications, or as part of a Larger Work; and + +under Patent Claims of such Contributor to make, use, sell, offer for sale, have +made, import, and otherwise transfer either its Contributions or its Contributor +Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution become +effective for each Contribution on the date the Contributor first distributes +such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under this +License. No additional rights or licenses will be implied from the distribution +or licensing of Covered Software under this License. Notwithstanding Section +2.1(b) above, no patent license is granted by a Contributor: + +for any code that a Contributor has removed from Covered Software; or + +for infringements caused by: (i) Your and any other third party’s +modifications of Covered Software, or (ii) the combination of its Contributions +with other software (except as part of its Contributor Version); or + +under Patent Claims infringed by Covered Software in the absence of its +Contributions. + +This License does not grant any rights in the trademarks, service marks, or +logos of any Contributor (except as may be necessary to comply with the notice +requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to distribute +the Covered Software under a subsequent version of this License (see Section +10.2) or under the terms of a Secondary License (if permitted under the terms of +Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its Contributions are +its original creation(s) or it has sufficient rights to grant the rights to its +Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under applicable +copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in +Section 2.1. + +3. Responsibilities + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under the +terms of this License. You must inform recipients that the Source Code Form of +the Covered Software is governed by the terms of this License, and how they can +obtain a copy of this License. You may not attempt to alter or restrict the +recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +such Covered Software must also be made available in Source Code Form, as +described in Section 3.1, and You must inform recipients of the Executable Form +how they can obtain a copy of such Source Code Form by reasonable means in a +timely manner, at a charge no more than the cost of distribution to the +recipient; and + +You may distribute such Executable Form under the terms of this License, or +sublicense it under different terms, provided that the license for the +Executable Form does not attempt to limit or alter the recipients’ rights in +the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, provided +that You also comply with the requirements of this License for the Covered +Software. If the Larger Work is a combination of Covered Software with a work +governed by one or more Secondary Licenses, and the Covered Software is not +Incompatible With Secondary Licenses, this License permits You to additionally +distribute such Covered Software under the terms of such Secondary License(s), +so that the recipient of the Larger Work may, at their option, further +distribute the Covered Software under the terms of either this License or such +Secondary License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices (including +copyright notices, patent notices, disclaimers of warranty, or limitations of +liability) contained within the Source Code Form of the Covered Software, except +that You may alter any license notices to the extent required to remedy known +factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, indemnity +or liability obligations to one or more recipients of Covered Software. However, +You may do so only on Your own behalf, and not on behalf of any Contributor. You +must make it absolutely clear that any such warranty, support, indemnity, or +liability obligation is offered by You alone, and You hereby agree to indemnify +every Contributor for any liability incurred by such Contributor as a result of +warranty, support, indemnity or liability terms You offer. You may include +additional disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + +If it is impossible for You to comply with any of the terms of this License with +respect to some or all of the Covered Software due to statute, judicial order, +or regulation then You must: (a) comply with the terms of this License to the +maximum extent possible; and (b) describe the limitations and the code they +affect. Such description must be placed in a text file included with all +distributions of the Covered Software under this License. Except to the extent +prohibited by statute or regulation, such description must be sufficiently +detailed for a recipient of ordinary skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You +fail to comply with any of its terms. However, if You become compliant, then the +rights granted under this License from a particular Contributor are reinstated +(a) provisionally, unless and until such Contributor explicitly and finally +terminates Your grants, and (b) on an ongoing basis, if such Contributor fails +to notify You of the non-compliance by some reasonable means prior to 60 days +after You have come back into compliance. Moreover, Your grants from a +particular Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the first +time You have received notice of non-compliance with this License from such +Contributor, and You become compliant prior to 30 days after Your receipt of the +notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, counter-claims, and +cross-claims) alleging that a Contributor Version directly or indirectly +infringes any patent, then the rights granted to You by any and all Contributors +for the Covered Software under Section 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user +license agreements (excluding distributors and resellers) which have been +validly granted by You or Your distributors under this License prior to +termination shall survive termination. + +6. Disclaimer of Warranty + +Covered Software is provided under this License on an “as is” basis, without +warranty of any kind, either expressed, implied, or statutory, including, +without limitation, warranties that the Covered Software is free of defects, +merchantable, fit for a particular purpose or non-infringing. The entire risk as +to the quality and performance of the Covered Software is with You. Should any +Covered Software prove defective in any respect, You (not any Contributor) +assume the cost of any necessary servicing, repair, or correction. This +disclaimer of warranty constitutes an essential part of this License. No use of +any Covered Software is authorized under this License except under this +disclaimer. + +7. Limitation of Liability + +Under no circumstances and under no legal theory, whether tort (including +negligence), contract, or otherwise, shall any Contributor, or anyone who +distributes Covered Software as permitted above, be liable to You for any +direct, indirect, special, incidental, or consequential damages of any character +including, without limitation, damages for lost profits, loss of goodwill, work +stoppage, computer failure or malfunction, or any and all other commercial +damages or losses, even if such party shall have been informed of the +possibility of such damages. This limitation of liability shall not apply to +liability for death or personal injury resulting from such party’s negligence +to the extent applicable law prohibits such limitation. Some jurisdictions do +not allow the exclusion or limitation of incidental or consequential damages, so +this exclusion and limitation may not apply to You. + +8. Litigation + +Any litigation relating to this License may be brought only in the courts of a +jurisdiction where the defendant maintains its principal place of business and +such litigation shall be governed by laws of that jurisdiction, without +reference to its conflict-of-law provisions. Nothing in this Section shall +prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + +This License represents the complete agreement concerning the subject matter +hereof. If any provision of this License is held to be unenforceable, such +provision shall be reformed only to the extent necessary to make it enforceable. +Any law or regulation which provides that the language of a contract shall be +construed against the drafter shall not be used to construe this License against +a Contributor. + +10. Versions of the License + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section 10.3, +no one other than the license steward has the right to modify or publish new +versions of this License. Each version will be given a distinguishing version +number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version of the +License under which You originally received the Covered Software, or under the +terms of any subsequent version published by the license steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to create a +new license for such software, you may create and use a modified version of this +License if you rename the license and remove any references to the name of the +license steward (except to note that such modified license differs from this +License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + +If You choose to distribute Source Code Form that is Incompatible With Secondary +Licenses under the terms of this version of the License, the notice described in +Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not distributed with this file, You can obtain one +at https://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + +This Source Code Form is “Incompatible With Secondary Licenses”, as defined +by the Mozilla Public License, v. 2.0. + +The binary distribution of this product bundles these dependencies under the +following license: +JDOM 1.1 +-------------------------------------------------------------------------------- +/*-- + + Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the disclaimer that follows + these conditions in the documentation and/or other materials + provided with the distribution. + + 3. The name "JDOM" must not be used to endorse or promote products + derived from this software without prior written permission. For + written permission, please contact . + + 4. Products derived from this software may not be called "JDOM", nor + may "JDOM" appear in their name, without prior written permission + from the JDOM Project Management . + + In addition, we request (but do not require) that you include in the + end-user documentation provided with the redistribution and/or in the + software itself an acknowledgement equivalent to the following: + "This product includes software developed by the + JDOM Project (http://www.jdom.org/)." + Alternatively, the acknowledgment may be graphical using the logos + available at http://www.jdom.org/images/logos. + + THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + + This software consists of voluntary contributions made by many + individuals on behalf of the JDOM Project and was originally + created by Jason Hunter and + Brett McLaughlin . For more information + on the JDOM Project, please see . + + */ + +The binary distribution of this product bundles these dependencies under the +following license: +Hbase Server 1.2.4 +-------------------------------------------------------------------------------- +This project bundles a derivative image for our Orca Logo. This image is +available under the Creative Commons By Attribution 3.0 License. + + Creative Commons Legal Code + + Attribution 3.0 Unported + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR + DAMAGES RESULTING FROM ITS USE. + + License + + THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE + COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY + COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS + AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + + BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE + TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY + BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS + CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND + CONDITIONS. + + 1. Definitions + + a. "Adaptation" means a work based upon the Work, or upon the Work and + other pre-existing works, such as a translation, adaptation, + derivative work, arrangement of music or other alterations of a + literary or artistic work, or phonogram or performance and includes + cinematographic adaptations or any other form in which the Work may be + recast, transformed, or adapted including in any form recognizably + derived from the original, except that a work that constitutes a + Collection will not be considered an Adaptation for the purpose of + this License. For the avoidance of doubt, where the Work is a musical + work, performance or phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be considered an + Adaptation for the purpose of this License. + b. "Collection" means a collection of literary or artistic works, such as + encyclopedias and anthologies, or performances, phonograms or + broadcasts, or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection and + arrangement of their contents, constitute intellectual creations, in + which the Work is included in its entirety in unmodified form along + with one or more other contributions, each constituting separate and + independent works in themselves, which together are assembled into a + collective whole. A work that constitutes a Collection will not be + considered an Adaptation (as defined above) for the purposes of this + License. + c. "Distribute" means to make available to the public the original and + copies of the Work or Adaptation, as appropriate, through sale or + other transfer of ownership. + d. "Licensor" means the individual, individuals, entity or entities that + offer(s) the Work under the terms of this License. + e. "Original Author" means, in the case of a literary or artistic work, + the individual, individuals, entity or entities who created the Work + or if no individual or entity can be identified, the publisher; and in + addition (i) in the case of a performance the actors, singers, + musicians, dancers, and other persons who act, sing, deliver, declaim, + play in, interpret or otherwise perform literary or artistic works or + expressions of folklore; (ii) in the case of a phonogram the producer + being the person or legal entity who first fixes the sounds of a + performance or other sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast. + f. "Work" means the literary and/or artistic work offered under the terms + of this License including without limitation any production in the + literary, scientific and artistic domain, whatever may be the mode or + form of its expression including digital form, such as a book, + pamphlet and other writing; a lecture, address, sermon or other work + of the same nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a musical + composition with or without words; a cinematographic work to which are + assimilated works expressed by a process analogous to cinematography; + a work of drawing, painting, architecture, sculpture, engraving or + lithography; a photographic work to which are assimilated works + expressed by a process analogous to photography; a work of applied + art; an illustration, map, plan, sketch or three-dimensional work + relative to geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of data to the + extent it is protected as a copyrightable work; or a work performed by + a variety or circus performer to the extent it is not otherwise + considered a literary or artistic work. + g. "You" means an individual or entity exercising rights under this + License who has not previously violated the terms of this License with + respect to the Work, or who has received express permission from the + Licensor to exercise rights under this License despite a previous + violation. + h. "Publicly Perform" means to perform public recitations of the Work and + to communicate to the public those public recitations, by any means or + process, including by wire or wireless means or public digital + performances; to make available to the public Works in such a way that + members of the public may access these Works from a place and at a + place individually chosen by them; to perform the Work to the public + by any means or process and the communication to the public of the + performances of the Work, including by public digital performance; to + broadcast and rebroadcast the Work by any means including signs, + sounds or images. + i. "Reproduce" means to make copies of the Work by any means including + without limitation by sound or visual recordings and the right of + fixation and reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or other electronic + medium. + + 2. Fair Dealing Rights. Nothing in this License is intended to reduce, + limit, or restrict any uses free from copyright or rights arising from + limitations or exceptions that are provided for in connection with the + copyright protection under copyright law or other applicable laws. + + 3. License Grant. Subject to the terms and conditions of this License, + Licensor hereby grants You a worldwide, royalty-free, non-exclusive, + perpetual (for the duration of the applicable copyright) license to + exercise the rights in the Work as stated below: + + a. to Reproduce the Work, to incorporate the Work into one or more + Collections, and to Reproduce the Work as incorporated in the + Collections; + b. to create and Reproduce Adaptations provided that any such Adaptation, + including any translation in any medium, takes reasonable steps to + clearly label, demarcate or otherwise identify that changes were made + to the original Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or a + modification could indicate "The original work has been modified."; + c. to Distribute and Publicly Perform the Work including as incorporated + in Collections; and, + d. to Distribute and Publicly Perform Adaptations. + e. For the avoidance of doubt: + + i. Non-waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the Licensor + reserves the exclusive right to collect such royalties for any + exercise by You of the rights granted under this License; + ii. Waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the Licensor waives the + exclusive right to collect such royalties for any exercise by You + of the rights granted under this License; and, + iii. Voluntary License Schemes. The Licensor waives the right to + collect royalties, whether individually or, in the event that the + Licensor is a member of a collecting society that administers + voluntary licensing schemes, via that society, from any exercise + by You of the rights granted under this License. + + The above rights may be exercised in all media and formats whether now + known or hereafter devised. The above rights include the right to make + such modifications as are technically necessary to exercise the rights in + other media and formats. Subject to Section 8(f), all rights not expressly + granted by Licensor are hereby reserved. + + 4. Restrictions. The license granted in Section 3 above is expressly made + subject to and limited by the following restrictions: + + a. You may Distribute or Publicly Perform the Work only under the terms + of this License. You must include a copy of, or the Uniform Resource + Identifier (URI) for, this License with every copy of the Work You + Distribute or Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or the ability of + the recipient of the Work to exercise the rights granted to that + recipient under the terms of the License. You may not sublicense the + Work. You must keep intact all notices that refer to this License and + to the disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or Publicly + Perform the Work, You may not impose any effective technological + measures on the Work that restrict the ability of a recipient of the + Work from You to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to the Work as + incorporated in a Collection, but this does not require the Collection + apart from the Work itself to be made subject to the terms of this + License. If You create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the Collection any credit + as required by Section 4(b), as requested. If You create an + Adaptation, upon notice from any Licensor You must, to the extent + practicable, remove from the Adaptation any credit as required by + Section 4(b), as requested. + b. If You Distribute, or Publicly Perform the Work or any Adaptations or + Collections, You must, unless a request has been made pursuant to + Section 4(a), keep intact all copyright notices for the Work and + provide, reasonable to the medium or means You are utilizing: (i) the + name of the Original Author (or pseudonym, if applicable) if supplied, + and/or if the Original Author and/or Licensor designate another party + or parties (e.g., a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's copyright notice, + terms of service or by other reasonable means, the name of such party + or parties; (ii) the title of the Work if supplied; (iii) to the + extent reasonably practicable, the URI, if any, that Licensor + specifies to be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information for the Work; + and (iv) , consistent with Section 3(b), in the case of an Adaptation, + a credit identifying the use of the Work in the Adaptation (e.g., + "French translation of the Work by Original Author," or "Screenplay + based on original Work by Original Author"). The credit required by + this Section 4 (b) may be implemented in any reasonable manner; + provided, however, that in the case of a Adaptation or Collection, at + a minimum such credit will appear, if a credit for all contributing + authors of the Adaptation or Collection appears, then as part of these + credits and in a manner at least as prominent as the credits for the + other contributing authors. For the avoidance of doubt, You may only + use the credit required by this Section for the purpose of attribution + in the manner set out above and, by exercising Your rights under this + License, You may not implicitly or explicitly assert or imply any + connection with, sponsorship or endorsement by the Original Author, + Licensor and/or Attribution Parties, as appropriate, of You or Your + use of the Work, without the separate, express prior written + permission of the Original Author, Licensor and/or Attribution + Parties. + c. Except as otherwise agreed in writing by the Licensor or as may be + otherwise permitted by applicable law, if You Reproduce, Distribute or + Publicly Perform the Work either by itself or as part of any + Adaptations or Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work which would be + prejudicial to the Original Author's honor or reputation. Licensor + agrees that in those jurisdictions (e.g. Japan), in which any exercise + of the right granted in Section 3(b) of this License (the right to + make Adaptations) would be deemed to be a distortion, mutilation, + modification or other derogatory action prejudicial to the Original + Author's honor and reputation, the Licensor will waive or not assert, + as appropriate, this Section, to the fullest extent permitted by the + applicable national law, to enable You to reasonably exercise Your + right under Section 3(b) of this License (right to make Adaptations) + but not otherwise. + + 5. Representations, Warranties and Disclaimer + + UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR + OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY + KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, + INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, + FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF + LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, + WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION + OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + + 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE + LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR + ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES + ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS + BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 7. Termination + + a. This License and the rights granted hereunder will terminate + automatically upon any breach by You of the terms of this License. + Individuals or entities who have received Adaptations or Collections + from You under this License, however, will not have their licenses + terminated provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will + survive any termination of this License. + b. Subject to the above terms and conditions, the license granted here is + perpetual (for the duration of the applicable copyright in the Work). + Notwithstanding the above, Licensor reserves the right to release the + Work under different license terms or to stop distributing the Work at + any time; provided, however that any such election will not serve to + withdraw this License (or any other license that has been, or is + required to be, granted under the terms of this License), and this + License will continue in full force and effect unless terminated as + stated above. + + 8. Miscellaneous + + a. Each time You Distribute or Publicly Perform the Work or a Collection, + the Licensor offers to the recipient a license to the Work on the same + terms and conditions as the license granted to You under this License. + b. Each time You Distribute or Publicly Perform an Adaptation, Licensor + offers to the recipient a license to the original Work on the same + terms and conditions as the license granted to You under this License. + c. If any provision of this License is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this License, and without further action + by the parties to this agreement, such provision shall be reformed to + the minimum extent necessary to make such provision valid and + enforceable. + d. No term or provision of this License shall be deemed waived and no + breach consented to unless such waiver or consent shall be in writing + and signed by the party to be charged with such waiver or consent. + e. This License constitutes the entire agreement between the parties with + respect to the Work licensed here. There are no understandings, + agreements or representations with respect to the Work not specified + here. Licensor shall not be bound by any additional provisions that + may appear in any communication from You. This License may not be + modified without the mutual written agreement of the Licensor and You. + f. The rights granted under, and the subject matter referenced, in this + License were drafted utilizing the terminology of the Berne Convention + for the Protection of Literary and Artistic Works (as amended on + September 28, 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 + and the Universal Copyright Convention (as revised on July 24, 1971). + These rights and subject matter take effect in the relevant + jurisdiction in which the License terms are sought to be enforced + according to the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. If the + standard suite of rights granted under applicable copyright law + includes additional rights not granted under this License, such + additional rights are deemed to be included in the License; this + License is not intended to restrict the license of any rights under + applicable law. + + + Creative Commons Notice + + Creative Commons is not a party to this License, and makes no warranty + whatsoever in connection with the Work. Creative Commons will not be + liable to You or any party on any legal theory for any damages + whatsoever, including without limitation any general, special, + incidental or consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, if Creative + Commons has expressly identified itself as the Licensor hereunder, it + shall have all rights and obligations of Licensor. + + Except for the limited purpose of indicating to the public that the + Work is licensed under the CCPL, Creative Commons does not authorize + the use by either party of the trademark "Creative Commons" or any + related trademark or logo of Creative Commons without the prior + written consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark usage + guidelines, as may be published on its website or otherwise made + available upon request from time to time. For the avoidance of doubt, + this trademark restriction does not form part of this License. + + Creative Commons may be contacted at https://creativecommons.org/. + +-------------------------------------------------------------------------------- + +For: /hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server +/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL + +xterm.js 3.8.0 +The source and binary distribution of this product bundles these dependencies +under the following license: + +Copyright (c) 2017-2018, The xterm.js authors (https://github.com/xtermjs/xterm.js) +Copyright (c) 2014-2016, SourceLair Private Company (https://www.sourcelair.com) +Copyright (c) 2012-2013, Christopher Jeffrey (https://github.com/chjj/) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/AbstractFuture.java and +hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/TimeoutFuture.java + +Copyright (C) 2007 The Guava Authors + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy of +the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. + +-------------------------------------------------------------------------------- + +Jline 3.9.0 +The binary distribution of this product bundles these dependencies under the +following license: + +Copyright (c) 2002-2018, the original author or authors. +All rights reserved. + +http://www.opensource.org/licenses/bsd-license.php + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + +Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with +the distribution. + +Neither the name of JLine nor the names of its contributors +may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, +BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/dev-support/submarine-installer/README-CN.md b/dev-support/submarine-installer/README-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..82a3baf4862d4c27c82e49b2c71e92d08f1b0e82 --- /dev/null +++ b/dev-support/submarine-installer/README-CN.md @@ -0,0 +1,228 @@ +# submarine installer + +## 项目介绍 + +介绍 **submarine-installer** 项目之前，首先要说明一下 **Submarine** 这个项目，**Submarine** 是 apache 顶级的机器学习平台项目。他将致力于支持 `Tensorflow`、`MXNet`、`Caffe`、`Spark` 等多种深度学习框架，提供了机器学习算法开发、分布式模型训练、模型管理和模型发布等全功能的系统框架，结合 hadoop 与身俱来的数据存储和数据处理能力，让数据科学家们能够更好的挖掘和发挥出数据的价值。 + +hadoop 在 2.9 版本中就已经让 YARN 支持了 Docker 容器的资源调度模式，**Submarine** 在此基础之上通过 YARN 把分布式深度学习框架以 Docker 容器的方式进行调度和运行起来。 + +由于分布式深度学习框架需要运行在多个 Docker 的容器之中，并且需要能够让运行在容器之中的各个服务相互协调，完成分布式机器学习的模型训练和模型发布等服务，这其中就会牵涉到 `DNS`、`Docker` 、 `GPU`、`Network`、`显卡`、`操作系统内核` 修改等多个系统工程问题，正确的部署好 **Hadoop {Submarine}** 的运行环境是一件很困难和耗时的事情。 + +为了降低 hadoop 2.9 以上版本的 docker 等组件的部署难度，所以我们专门开发了这个用来部署 `Submarine` 运行时环境的 `submarine-installer` 项目，提供一键安装脚本，也可以分步执行安装、卸载、启动和停止各个组件，同时讲解每一步主要参数配置和注意事项。我们同时提供了 [中文手册](project/github/submarine/docs/userdocs/yarn/InstallationGuideChineseVersion.md) 和 [英文手册](project/github/submarine/docs/userdocs/yarn/InstallationGuide.md) ，帮助用户更容易的部署，发现问题也可以及时解决。 + +## 先决条件 + +**submarine-installer** 目前只支持 `centos-release-7-3.1611.el7.centos.x86_64` 以上版本的操作系统中进行使用。 + +## 配置说明 + +使用 **submarine-installer** 进行部署之前，你可以参考 [install.conf](install.conf) 文件中已有的配置参数和格式，根据你的使用情况进行如下的参数配置： + ++ **DNS 配置项** + + LOCAL_DNS_HOST：服务器端本地 DNS IP 地址配置，可以从 `/etc/resolv.conf` 中查看 + + YARN_REGISTRY_DNS_HOST：yarn dns server 启动的 IP 地址 + ++ **ETCD 配置项** + + 机器学习是一个计算密度型系统，对数据传输性能要求非常高，所以我们使用了网络效率损耗最小的 ETCD 网络组件，它可以通过 BGP 路由方式支持 overlay 网络，同时在跨机房部署时支持隧道模式。 + + 你需要选择至少三台以上的服务器作为 ETCD 的运行服务器，这样可以让 `Submarine` 有较好的容错性和稳定性。 + + 在 **ETCD_HOSTS** 配置项中输入作为 ETCD 服务器的IP数组，参数配置一般是这样： + + ETCD_HOSTS=(hostIP1 hostIP2 hostIP3)，注意多个 hostIP 之间请使用空格进行隔开。 + ++ **DOCKER_REGISTRY 配置项** + + 你首先需要安装好一个可用的 docker 的镜像管理仓库，这个镜像仓库用来存放你所需要的各种深度学习框架的镜像文件，然后将镜像仓库的 IP 地址和端口配置进来，参数配置一般是这样：DOCKER_REGISTRY="10.120.196.232:5000" + ++ **DOWNLOAD_SERVER 配置项** + + `submarine-installer` 默认都是从网络上直接下载所有的依赖包（例如：GCC、Docker、Nvidia 驱动等等），这往往需要消耗大量的时间，并且在有些服务器不能连接互联网的环境中将无法部署，所以我们在 `submarine-installer` 中内置了 HTTP 下载服务，只需要在一台能够连接互联网的服务器中运行 `submarine-installer` ，就可以为所有其他服务器提供依赖包的下载，只需要你按照以下配置进行操作： + + 1. 首先，你需要将 `DOWNLOAD_SERVER_IP` 配置为一台能够连接互联网的服务器IP地址，将 `DOWNLOAD_SERVER_PORT` 配置为一个不会不太常用的端口。 + 2. 在 `DOWNLOAD_SERVER_IP` 所在的那台服务器中运行 `submarine-installer/install.sh` 命令后，在安装界面中选择 `[start download server]` 菜单项，`submarine-installer` 将会把部署所有的依赖包全部下载到 `submarine-installer/downloads` 目录中，然后通过 `python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT}` 命令启动一个 HTTP 下载服务，不要关闭这台服务器中运行着的 `submarine-installer` 。 + 3. 在其他服务器中同样运行 `submarine-installer/install.sh` 命令 ，按照安装界面中的 `[install component]` 菜单依次进行各个组件的安装时，会自动从 `DOWNLOAD_SERVER_IP` 所在的那台服务器下载依赖包进行安装部署。 + 4. **DOWNLOAD_SERVER** 另外还有一个用处是，你可以自行把各个依赖包手工下载下来，然后放到其中一台服务器的 `submarine-installer/downloads` 目录中，然后开启 `[start download server]` ，这样就可以为整个集群提供离线安装部署的能力。 + ++ **YARN_CONTAINER_EXECUTOR_PATH 配置项** + + 如何编译 YARN 的 container-executor：你进入到 `hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager` 目录中执行 `mvn package -Pnative -DskipTests` 命令，将会编译出 `./target/native/target/usr/local/bin/container-executor` 文件。 + + 你需要将 `container-executor` 文件的完整路径填写在 YARN_CONTAINER_EXECUTOR_PATH 配置项中。 + ++ **YARN_HIERARCHY 配置项** + + 请保持和你所使用的 YARN 集群的 `yarn-site.xml` 配置文件中的 `yarn.nodemanager.linux-container-executor.cgroups.hierarchy` 相同的配置，`yarn-site.xml` 中如果未配置该项，那么默认为 `/hadoop-yarn`。 + ++ **YARN_NODEMANAGER_LOCAL_DIRS 配置项** + + 请保持和你所使用的 YARN 集群的 `yarn-site.xml` 配置文件中的 `yarn.nodemanager.local-dirs` 相同的配置。 + ++ **YARN_NODEMANAGER_LOG_DIRS 配置项** + + 请保持和你所使用的 YARN 集群的 `yarn-site.xml` 配置文件中的 `yarn.nodemanager.log-dirs` 相同的配置。 + +## 使用说明 + +**submarine-installer** 完全使用 Shell 脚本编写，不需要安装 ansible 等任何部署工具，避免了不同公司用户的服务器管理规范不同而导致程序不通用，例如：有些机房是不容许 ROOT 用户通过 SHELL 直接进行远程服务器操作等。 + +**submarine-installer** 的部署过程，完全是通过在菜单中进行选择的操作方式进行的，避免了误操作的同时，你还可以通过各个菜单项目对任意一个组件进行分步执行安装、卸载、启动和停止各个组件，具有很好的灵活性，在部分组件出现问题后，也可以通过 **submarine-installer** 对系统进行诊断和修复。 + +**submarine-installer** 部署过程中屏幕中会显示日志信息，日志信息一共有三种字体颜色： + ++ 红色字体颜色：说明组件安装出现了错误，部署已经终止。 + ++ 绿色文字颜色：说明组件安装正常，部署正常运行。 + ++ 蓝色文字颜色：需要你按照提示信息在另外一个 SHELL 终端中进行手工输入命令，一般是修改操作系统内核配置操作，按照提示信息依次操作就可以了。 + +**启动 submarine-installer** + +运行 `submarine-installer/install.sh` 命令启动，部署程序首先会检测服务器中的网卡 IP 地址，如果服务器有多个网卡或配置了多个 IP ，会以列表的形式显示，选择你实际使用的 IP 地址。 + +**submarine-installer** 菜单说明： + +![alt text](./assets/submarine-installer.gif "Submarine Installer") + +## 部署说明 + +部署流程如下所示： + +1. 参照配置说明，根据你的服务器使用情况配置好 install.conf 文件 + +2. 将整个 `submarine-installer` 文件夹打包复制到所有的服务器节点中 + +3. 首先在配置为 **DOWNLOAD_SERVER** 的服务器中 + + + 运行 `submarine-installer/install.sh` 命令 + + + 在安装界面中选择 `[start download server]` 菜单项，等待下载完各个依赖包后，启动 HTTP 服务 + +4. 在其他需要进行部署的服务器中 + + 运行 `submarine-installer/install.sh` 命令，显示的主菜单 **[Main menu]** 中有以下菜单： + + + prepare system environment + + install component + + uninstall component + + start component + + stop component + + start download server + +5. **prepare system environment** + + + **prepare operation system** + + 检查部署服务器的操作系统和版本； + + + **prepare operation system kernel** + + 显示操作系统内核更新的操作命令的提示信息，根据你的选择是否自动更新内核版本； + + + **prepare GCC version** + + 显示操作系统中现在的 GCC 版本内核更新的操作命令的提示信息和根据你的选择是否自动更新 GCC 版本； + + + **check GPU** + + 检查服务器是否能够检测到 GPU 显卡； + + + **prepare user&group** + + 显示添加 hadoop 和 docker 的用户和用户组操作命令的提示信息，需要你自己根据提示信息检查服务器中是否存在所需要的用户和用户组； + + + **prepare nvidia environment** + + 自动进行操作系统内核和头文件的更新，自动安装 `epel-release` 和 `dkms` ； + + 显示修改系统内核参数配置的操作命令的提示信息，需要你另外打开一个终端根据命令顺序执行； + +6. install component + + + **install etcd** + + 下载 etcd 的 bin 文件，并安装到 `/usr/bin` 目录中； + + 根据 **ETCD_HOSTS** 配置项生成 `etcd.service` 文件， 安装到 `/etc/systemd/system/` 目录中； + + + **install docker** + + 下载 docker 的 RPM 包进行本地安装； + + 生成 `daemon.json` 配置文件，安装到 `/etc/docker/` 目录中； + + 生成 `docker.service` 配置文件，安装到 `/etc/systemd/system/` 目录中； + + + **install calico network** + + 下载 `calico` 、`calicoctl` 和 `calico-ipam` 文件，安装到 `/usr/bin` 目录中； + + 生成 `calicoctl.cfg` 配置文件，安装到 `/etc/calico/` 目录中； + + 生成 `calico-node.service` 配置文件，安装到 `/etc/systemd/system/` 目录中； + + 安装完毕后，会在容器中会根据 **CALICO_NETWORK_NAME** 配置项自动创建 calico network，并自动创建 2 个 Docker 容器，检查 2 个容器是否能偶互相 PING 通； + + + **install nvidia driver** + + 下载 `nvidia-detect` 文件，在服务器中检测显卡版本； + + 根据显卡版本号下载 Nvidia 显卡驱动安装包； + + 检测本服务器中是否 `disabled Nouveau` ，如果没有停止安装，那么你需要执行 **[prepare system environment]** 菜单中的 **[prepare nvidia environment]** 子菜单项，按照提示进行操作； + + 如果本服务器中已经 `disabled Nouveau` ，那么就会进行本地安装； + + + **install nvidia docker** + + 下载 `nvidia-docker` 的 RPM 安装包并进行安装； + + 显示检测 `nvidia-docker` 是否可用的命令提示信息，需要你另外打开一个终端根据命令顺序执行； + + + **install yarn container-executor** + + 根据 **YARN_CONTAINER_EXECUTOR_PATH 配置项**，将 `container-executor` 文件复制到 `/etc/yarn/sbin/Linux-amd64-64/` 目录中； + + 根据配置生成 `container-executor.cfg` 文件，复制到 `/etc/yarn/sbin/etc/hadoop/` 目录中； + + + **install submarine autorun script** + + 复制 `submarine.sh` 文件到 `/etc/rc.d/init.d/` 目录中； + + 将 `/etc/rc.d/init.d/submarine.sh` 添加到 `/etc/rc.d/rc.local` 系统自启动文件中； + +7. uninstall component + + 删除指定组件的 BIN 文件和配置文件，不在复述 + + - uninstall etcd + - uninstall docker + - uninstall calico network + - uninstall nvidia driver + - uninstall nvidia docker + - uninstall yarn container-executor + - uninstall submarine autorun script + +8. start component + + 重启指定组件，不在复述 + + - start etcd + - start docker + - start calico network + +9. stop component + + 停止指定组件，不在复述 + + - stop etcd + - stop docker + - stop calico network + +10. start download server + + 只能在 **DOWNLOAD_SERVER_IP 配置项** 所在的服务器中才能执行本操作； + diff --git a/dev-support/submarine-installer/README.md b/dev-support/submarine-installer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9acd420f69bbd70c8a68cc9d4ff0053af6328e36 --- /dev/null +++ b/dev-support/submarine-installer/README.md @@ -0,0 +1,236 @@ + +# submarine installer + +## Introduction + +Submarine is the latest machine learning framework. It aims to support `Tensorflow`, `MXNet`,` Caffe`, `Spark`, etc. A variety of deep learning frameworks provide a full-featured system framework for machine learning algorithm development, distributed model training, model management, and model publishing, combined with hadoop's intrinsic data storage and data processing capabilities to enable data scientists to Good mining and the value of the data. + +Hadoop has enabled YARN to support Docker container since 2.x. **Submarine** then uses YARN to schedule and run the distributed deep learning framework in the form of a Docker container. + +Since the distributed deep learning framework needs to run in multiple Docker containers and needs to be able to coordinate the various services running in the container, complete the services of model training and model publishing for distributed machine learning. Involving multiple system engineering problems such as `DNS`, `Docker`, `GPU`, `Network`, `graphics card`, `operating system kernel` modification, etc. It is very difficult and time-consuming to properly deploy the **Submarine** runtime environment. + +In order to reduce the difficulty of deploying components, we have developed this **submarine-installer** project to deploy the **Submarine** runtime environment, providing a one-click installation script or step-by-step installation. Unload, start, and stop individual components, and explain the main parameter configuration and considerations for each step. We also provides a [Chinese manual](project/github/submarine/docs/userdocs/yarn/InstallationGuideChineseVersion.md) and an [English manual](project/github/submarine/docs/userdocs/yarn/InstallationGuide.md) for the **Submarine** runtime environment to help users deploy more easily and find problems in a timely manner. + +This installer is just created for your convenience and for test purpose only. You can choose to install required libraries by yourself, please don't run this script in your production environment before fully validate it in a sandbox environment. + +## prerequisites + +**submarine-installer** currently only supports operating systems based on `centos-release-7-3.1611.el7.centos.x86_64` and above. + +## Configuration instructions + +Before deploying with submarine-installer, you can refer to the existing configuration parameters and format in the `install.conf` file, and configure the following parameters according to your usage: + ++ **DNS Configuration** + + LOCAL_DNS_HOST: server-side local DNS IP address configuration, which can be viewed from `/etc/resolv.conf` + + YARN_REGISTRY_DNS_HOST: yarn dns server started IP address + ++ **ETCD Configuration** + + Machine learning is a computationally-density system that requires very high data transmission performance. Therefore, we use the ETCD network component with the least network efficiency loss. It can support the overlay network through BGP routing and support tunnel mode when deployed across the equipment room. + + Please note that you can choose to use different Docker networks. ETCD is not the only network solution supported by Submarine. + + You need to select at least three servers as the running server for ETCD, which will make **Submarine** better fault tolerant and stable. + + Enter the IP array as the ETCD server in the ETCD_HOSTS configuration item. The parameter configuration is generally like this: + + ETCD_HOSTS=(hostIP1 hostIP2 hostIP3). Note that spaces between multiple hostIPs should be separated by spaces. + ++ **DOCKER_REGISTRY Configuration** + + You can follow the following step to setup your Docker registry. But it is not a hard requirement since you can use a pre-setup Docker registry instead. + + You first need to install an image management repository for the available docker. This image repository is used to store the image files of the various deep learning frameworks you need, and then configure the IP address and port of the mirror repository. The parameter configuration is generally the same : + + DOCKER_REGISTRY="10.120.196.232:5000" + ++ **DOWNLOAD_SERVER Configuration** + + By default, **submarine-installer** downloads all dependencies directly from the network (eg GCC, Docker, Nvidia drivers, etc.), which often takes a lot of time and cannot be used in environments where some servers cannot connect to the Internet. Deployment, so we built the HTTP download service in **submarine-installer**, you only need to run **submarine-installer** on a server that can connect to the Internet, you can download the dependencies for all other servers, you only need Follow these configurations: + + 1. First, you need to configure `DOWNLOAD_SERVER_IP` as a server IP address that can connect to the Internet, and configure `DOWNLOAD_SERVER_PORT` as a port that is not very common. + 2. After running the `submarine-installer/install.sh` command on the server where `DOWNLOAD_SERVER_IP` is located, select the `[start download server]` menu item in the installation interface. **submarine-installer** will download all the dependencies of the deployment to the server. In the `submarine-installer/downloads` directory, start an HTTP download service with the `python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT}` command. Do not close the **submarine-installer** running on this server. + 3. When you run the `submarine-installer/install.sh` command on other servers and follow the `[install component]` menu in the installation interface to install each component in turn, it will automatically download the dependencies from the server where `DOWNLOAD_SERVER_IP` is located for installation and deployment. . + 4. **DOWNLOAD_SERVER** Another useful thing is that you can manually download the dependencies by hand, put them in the `submarine-installer/downloads` directory of one of the servers, and then open `[start download server]`, so that you can The cluster provides the ability to deploy offline deployments. + ++ **YARN_CONTAINER_EXECUTOR_PATH Configuration** + + You can get container-executor binary from either binary release package or build from source. + You need to fill in the full path of the container-executor file in the `YARN_CONTAINER_EXECUTOR_PATH` configuration item. + ++ **YARN_HIERARCHY Configuration** + + Please keep the same configuration as `yarn.nodemanager.linux-container-executor.cgroups.hierarchy` in the `yarn-site.xml` configuration file of the YARN cluster you are using. If this is not configured in `yarn-site.xml`, Then the default is `/hadoop-yarn`. + ++ **YARN_NODEMANAGER_LOCAL_DIRS Configuration** + + Please keep the same configuration as `yarn.nodemanager.local-dirs` in the `yarn-site.xml` configuration file of the YARN cluster you are using. + ++ **YARN_NODEMANAGER_LOG_DIRS Configuration** + + Please keep the same configuration as `yarn.nodemanager.log-dirs` in the `yarn-site.xml` configuration file of the YARN cluster you are using. + +## Instructions for use + +**submarine-installer** is completely written in shell script. It does not need to install any deployment tools such as ansible. It avoids different server management specifications of different company users and causes the program to be uncommon. For example, some computer rooms do not allow ROOT users to directly remotely through SHELL. Server operation, etc. + +The deployment process of **submarine-installer** is completely performed by selecting the operation in the menu. It avoids misoperations. You can also install, uninstall, and start any component in each step through various menu items. And the various components are stopped, and the flexibility is very good. After some components have problems, the system can also be diagnosed and repaired by **submarine-installer**. + +**submarine-installer** The log information is displayed on the screen during the deployment process. The log information has three font colors: + ++ Red font color: Indicates that the component installation has an error and the deployment has terminated. + ++ Green text color: The component is installed properly and the deployment is working properly. + ++ Blue text color: You need to manually enter the command in another SHELL terminal according to the prompt information. Generally, modify the operating system kernel configuration operation, and follow the prompt information to operate it. + +**Start submarine-installer** + +Run the `submarine-installer/install.sh` command to start. The deployment program first detects the IP address of the network card in the server. If the server has multiple network cards or multiple IP addresses configured, it will be displayed in the form of a list. Select the one you actually use. IP address. + +**submarine-installer** Menu description： + +![alt text](./assets/submarine-installer.gif "Submarine Installer") + +## Deployment instructions + +The deployment process is as follows: + +1. Refer to the configuration instructions to configure the `install.conf` file based on your server usage. + +2. Copy the entire **submarine-installer** folder to all server nodes + +3. First in the server configured as **DOWNLOAD_SERVER** + + + Run the `submarine-installer/install.sh` command + + + Select the `[start download server]` menu item in the installation interface, and wait for the download of each dependency package to start the HTTP service. + +4. **In other servers that need to be deployed** + + Run the `submarine-installer/install.sh` command to display the following menu in the main menu **[Main menu]**: + + + prepare system environment + + install component + + uninstall component + + start component + + stop component + + start download server + +5. **prepare system environment** + + - **prepare operation system** + + Check the operating system and version of the deployment server; + + - **prepare operation system kernel** + + Display the prompt information of the operation command of the operating system kernel update, and automatically update the kernel version according to your choice; + + - **prepare GCC version** + + Display the prompt information of the operation command of the current GCC version kernel update in the operating system and whether to automatically update the GCC version according to your choice; + + - **check GPU** + + Check if the server can detect the GPU graphics card; + + - **prepare user&group** + + Display the prompts for adding user and user group operation commands for hadoop and docker. You need to check whether there are any required users and user groups in the server according to the prompt information. + + - **prepare nvidia environment** + + Automatically update the operating system kernel and header files, and automatically install `epel-release` and `dkms`; + + Display the prompt information for modifying the operation command of the system kernel parameter configuration, you need to open another terminal according to the command sequence; + +6. **install component** + + - **install etcd** + + Download the bin file for etcd and install it in the `/usr/bin` directory; + + Generate the `etcd.service` file according to the **ETCD_HOSTS** configuration item and install it into the `/etc/systemd/system/` directory. + + - **install docker** + + Download docker's RPM package for local installation; + + Generate the `daemon.json` configuration file and install it into the `/etc/docker/` directory. + + Generate the `docker.service` configuration file and install it into the `/etc/systemd/system/` directory. + + - **install calico network** + + Download the `calico`, `calicoctl`, and `calico-ipam` files and install them in the `/usr/bin` directory. + + Generate the `calicoctl.cfg` configuration file and install it into the `/etc/calico/` directory. + + Generate the `calico-node.service` configuration file and install it into the `/etc/systemd/system/` directory. + + After the installation is complete, the calico network will be automatically created in the container according to the **CALICO_NETWORK_NAME** configuration item, and two Docker containers will be created automatically to check whether the two containers can even ping each other. + + - **install nvidia driver** + + Download the `nvidia-detect` file to detect the graphics card version in the server;Download the `nvidia-detect` file to detect the graphics card version in the server; + + Download the Nvidia graphics driver installation package according to the graphics card version number; + + Check if the Nouveau is disabled in this server. If the installation is not stopped, you need to execute the **[prepare nvidia environment]** submenu item in the **[prepare system environment]** menu and follow the prompts. + + If Nouveau has been disabled in this server, it will be installed locally; + + - **install nvidia docker** + + Download the nvidia-docker RPM installation package and install it; + + Display the command prompt information to detect whether nvidia-docker is available. You need to open another terminal to execute according to the command sequence. + + - **install yarn container-executor** + + Copy the `container-executor` file to the `/etc/yarn/sbin/Linux-amd64-64/` directory according to the **YARN_CONTAINER_EXECUTOR_PATH** configuration item; + + Generate the `container-executor.cfg` file according to the configuration and copy it to the `/etc/yarn/sbin/etc/hadoop/` directory. + + - **install submarine autorun script** + + Copy the submarine.sh file to the `/etc/rc.d/init.d/` directory; + + Add `/etc/rc.d/init.d/submarine.sh` to the `/etc/rc.d/rc.local` system self-starting file; + +7. uninstall component + + Delete the BIN file and configuration file of the specified component, not in the retelling + + - uninstall etcd + - uninstall docker + - uninstall calico network + - uninstall nvidia driver + - uninstall nvidia docker + - uninstall yarn container-executor + - uninstall submarine autorun script + +8. start component + + Restart the specified component, not repeat + + - start etcd + - start docker + - start calico network + +9. stop component + + Stop specifying component, not repeating + + - stop etcd + - stop docker + - stop calico network + +10. start download server + + This operation can only be performed on the server where the **DOWNLOAD_SERVER_IP** configuration item is located; + diff --git a/dev-support/submarine-installer/assets/submarine-installer.gif b/dev-support/submarine-installer/assets/submarine-installer.gif new file mode 100644 index 0000000000000000000000000000000000000000..56b3b690f0ad4fd4f4d21c461bee6e1714070fbf Binary files /dev/null and b/dev-support/submarine-installer/assets/submarine-installer.gif differ diff --git a/dev-support/submarine-installer/install.conf b/dev-support/submarine-installer/install.conf new file mode 100644 index 0000000000000000000000000000000000000000..69a7dbc46771696e48e43626a723c58f29f019a1 --- /dev/null +++ b/dev-support/submarine-installer/install.conf @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ DOWNLOAD CONFIGURATION #################################################### +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve the problem that all servers are slow to download online. +# At the same time, you can also manually download the dependencies to the downloads directory. +# Offline installation of the system +# The host ip on which download service is started. +DOWNLOAD_SERVER_IP="10.196.69.175" +DOWNLOAD_SERVER_PORT="19000" + +################################ DNS CONFIGURATION ################################################# +# The default DNS ip for the cluster +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +# see YARN_REGISTRY_DNS_HOST + +# determine whether install an security yarn ,value is "true" or "false",default is "true" +YARN_SECURITY="true" + +################################# YARN CONFIGURATION ############################################# +# The resourcemanager hosts. For example, YARN_RESOURCE_MANAGER_HOSTS=(hostname1 hostname2) +YARN_RESOURCE_MANAGER_HOSTS=() +# The hosts should not install nodemanager. Multiple hosts should be split by spaces. For example, YARN_NODE_MANAGER_EXCLUDE_HOSTS=(hostname1 hostname2) +YARN_NODE_MANAGER_EXCLUDE_HOSTS=() +# hadoop.registry.dns.bind-port & hadoop.registry.dns.bind-address in yarn-site.xml +YARN_REGISTRY_DNS_HOST="" # yarn registry dns host +YARN_REGISTRY_DNS_HOST_PORT="53" +YARN_TIMELINE_HOST="" # timeline-v1.5 & v2.0 +YARN_JOB_HISTORY_HOST="" +YARN_SPARK_HISTORY_HOST="" + +# Kerberos realm used in core-site.xml +LOCAL_REALM="" +# The namenode hosts. For example, HDFS_NAMENODE_HOSTS=(hostname1 hostname2) +HDFS_NAMENODE_HOSTS=() + +# fs.defaultFS in core-site.xml. For example, FS_DEFAULTFS="hdfs://machine-learning" +FS_DEFAULTFS="" + +# local.cluster-id in yarn-site.xml +LOCAL_CLUSTER_ID="ml-yarn" + +# yarn.app.mapreduce.am.staging-dir in mapred-site.xml +YARN_APP_MAPREDUCE_AM_STAGING_DIR="/tmp/ml-yarn-staging" + +# yarn container-executor config +# How to compile container-executor: +# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager +# path and enter the command: `mvn package -Pnative -DskipTests` +# Only the nodemanager is compiled. The path of the compiled container-executor is: +# ./target/native/target/usr/local/bin/container-executor +YARN_CONTAINER_EXECUTOR_PATH="/etc/yarn/sbin/Linux-amd64-64/container-executor" + +# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy' +# in yarn-site.xml, default '/hadoop-yarn' +YARN_HIERARCHY="/hadoop-yarn" + +# The home paths for container dir +YARN_NODEMANAGER_LOCAL_HOME_PATHS="/home/hadoop/disk/1/yarn,/home/hadoop/disk/2/yarn,/home/hadoop/disk/3/yarn" + +# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml +YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local" + +# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml +YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs" + +# Create '/app-logs' path on hdfs, Owner is 'yarn', group is 'hadoop', +# and 'hadoop' group needs to include 'hdfs, yarn, mapred' yarn-site.xml users, etc. +# `yarn.nodemanager.remote-app-log-dir` in yarn-site.xml +YARN_AGGREGATED_LOG_DIR="/ml-app-logs" + +# time line v1.5 store path on hdfs +YARN_TIMELINE_FS_STORE_DIR="/tmp/ml-entity-file-history" + +# hbase client +YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE="file:/home/hadoop/hbase-current/conf/hbase-site.xml" + +# yarn.resourcemanager.zk-address in yarn-site.xml +# hadoop.registry.zk.quorum in core-site.xml. For Example, YARN_ZK_ADDRESS="zookeeperHost1:2181,zookeeperHost2:2181,zookeeperHost3:2181" +YARN_ZK_ADDRESS="" + +# yarn.timeline-service.leveldb-state-store.path in yarn-site.xml +YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH="/home/hadoop/timeline" + +# yarn.nodemanager.recovery.dir in yarn-site.xml +YARN_NODEMANAGER_RECOVERY_DIR="/home/hadoop/yarn/yarn-nm-recovery" + +# yarn.nodemanager.runtime.linux.docker.default-container-network in yarn-site.xml +YARN_DOCKER_CONTAINER_DEFAULT_NETWORK=calico-network + +# docker.allowed-container-networks in yarn-site.xml and container-executor.cfg +YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS=host,none,bridge,calico-network + +YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH="/home/hadoop/hosts/yarn_exclude" + +YARN_GC_LOG_DIR="/home/hadoop/yarn/gclog" +YARN_PID_DIR="/home/hadoop/yarn/pid" +YARN_LOG_DIR="/home/hadoop/yarn/log" + +################################### YARN-kerberos CONFIGURATION ############################################## +# Kerberos authentication to create some necessary hdfs paths +HADOOP_KEYTAB_LOCATION="" +HADOOP_PRINCIPAL="" + +MAPRED_KEYTAB_LOCATION="/home/hadoop/yarn/conf/mapred.service.keytab" + +# yarn.timeline-service.keytab in yarn-site.xml +YARN_KEYTAB_LOCATION="/home/hadoop/yarn/conf/yarn.service.keytab" + +HTTP_KEYTAB_LOCATION="/home/hadoop/yarn/conf/http.service.keytab" + +################################### JAVA_HOME CONFIGURATION ############################################## +JAVA_VERSION=jdk1.8.0_152 +JAVA_TARBALL=${JAVA_VERSION}.tgz +JAVA_HOME=/home/hadoop/java-current + +################################### YARN-hadoop CONFIGURATION ############################################## +HADOOP_SETUP_USER="hadoop" +HADOOP_VERSION="hadoop-3.3.0-SNAPSHOT" +HADOOP_TARBALL="${HADOOP_VERSION}.tar.gz" +HADOOP_HOME="/home/hadoop/hadoop-current" + +# hadoop.http.authentication.signature.secret.file in core-site.xml +HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE="/home/hadoop/yarn/conf/http-secret" + +################################# YARN-hbase CONFIGURATION ############################################## +# copy timeline hbase jar to ${hbase_client]/lib path +HBASE_HOME="/home/hadoop/hbase-current" + +################################ ETCD CONFIGURATION #################################################### +## etcd host ip list +ETCD_HOSTS=(10.196.69.201 10.196.69.173 10.196.69.174) + +ETCD_VERSION="v3.3.9" +ETCD_TAR_GZ="etcd-${ETCD_VERSION}-linux-amd64.tar.gz" +ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/${ETCD_TAR_GZ}" + +################################## CALICO CONFIGURATION ########################################## +CALICO_IPV4POOL_CIDR="192.20.0.0" +CALICO_NETWORK_NAME="calico-network" +CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico" +CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam" +CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl" + +################################ DOCKER CONFIG SEGMENT ################################################### +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" +# /var/lib/docker is used if DOCKER_STORE_PATH not specified +DOCKER_STORE_PATH= + +##### System component download url address, Generally do not need to be modified ##### +DOCKER_REPO="https://download.docker.com/linux/centos/7/x86_64/stable/Packages/" +DOCKER_VERSION="docker-ce" +DOCKER_VERSION_NUM="18.06.1.ce" +DOCKER_ENGINE_RPM="${DOCKER_VERSION}-${DOCKER_VERSION_NUM}-3.el7.x86_64.rpm" + +################################# NVIDIA CONFIGURATION ############################################# +NVIDIA_DETECT_URL="http://mirror.rackspace.com/elrepo/elrepo/el7/x86_64/RPMS" +# NVIDIA_DRIVER_VERSION can be used by download server to decide which version of +# nvidia driver should be download +NVIDIA_DRIVER_VERSION="" +NVIDIA_DOCKER_VERSION="2.0.3" +NVIDIA_DOCKER_GIT_SNAPSHOT_URL="https://api.github.com/repos/nvidia/" diff --git a/dev-support/submarine-installer/install.sh b/dev-support/submarine-installer/install.sh new file mode 100755 index 0000000000000000000000000000000000000000..93691b0c3fae79880883f337ce824b932d36e7b6 --- /dev/null +++ b/dev-support/submarine-installer/install.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# description: submarine install scripts. + +ROOT=$(cd "$(dirname "$0")"; pwd) +SUBMARINE_INSTALLER_VERSION="v0.8.0" +PACKAGE_DIR=${ROOT}/package +SCRIPTS_DIR=${ROOT}/scripts +INSTALL_TEMP_DIR=${ROOT}/temp +DOWNLOAD_DIR=${ROOT}/downloads +DATE=`date +%Y%m%d-%H:%M:%S` +LOG=${ROOT}/logs/install.log.`date +%Y%m%d%H%M%S` +LOCAL_HOST_IP_LIST=() +LOCAL_HOST_IP="" +OPERATING_SYSTEM="" +DOWNLOAD_HTTP="" + +# import shell script +. ${ROOT}/install.conf +. ${ROOT}/scripts/calico.sh +. ${ROOT}/scripts/docker.sh +. ${ROOT}/scripts/download-server.sh +. ${ROOT}/scripts/environment.sh +. ${ROOT}/scripts/etcd.sh +. ${ROOT}/scripts/yarn.sh +. ${ROOT}/scripts/yarn_insecure.sh +. ${ROOT}/scripts/menu.sh +. ${ROOT}/scripts/nvidia.sh +. ${ROOT}/scripts/nvidia-docker.sh +. ${ROOT}/scripts/submarine.sh +. ${ROOT}/scripts/utils.sh + +#================================= Main ======================================== +mkdir $ROOT/logs/ -p +mkdir $INSTALL_TEMP_DIR -p +mkdir $DOWNLOAD_DIR -p + +source /etc/os-release +OPERATING_SYSTEM=$ID + +get_ip_list +ipCount=${#LOCAL_HOST_IP_LIST[@]} +if [[ $ipCount -eq 1 ]]; then + LOCAL_HOST_IP=${LOCAL_HOST_IP_LIST[0]} +else + echo -e "Detect the network card IP in the server, \e[31m[${LOCAL_HOST_IP_LIST[@]}]\e[0m" + echo -n -e "please enter a valid IP address: " + + read ipInput + if ! valid_ip $ipInput; then + echo -e "you input \e[31m$ipInput\e[0m address format is incorrect! " | tee -a $LOG + exit_install + else + LOCAL_HOST_IP=$ipInput + fi +fi + +echo -n -e "Please confirm whether the IP address of this machine is \e[31m${LOCAL_HOST_IP}\e[0m?[y|n]" +read myselect +if [[ "$myselect" != "y" && "$myselect" != "Y" ]]; then + exit_install +fi + +check_install_conf + +if [[ -n "$DOWNLOAD_SERVER_IP" && -n "$DOWNLOAD_SERVER_PORT" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then + DOWNLOAD_HTTP="http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}" +fi + +check_install_user + +# Clean up the installation temporary directory +rm $INSTALL_TEMP_DIR/* -rf + + +menu_index="0" +for ((j=1;;j++)) +do + menu + case "$menu_index" in + "0") + menu_index="$menu_choice" + ;; + "1"|"2"|"3"|"4"|"5"|"6"|"7"|"8") + # echo "aaaa=$menu_index-$menu_choice" + menu_process + if [[ $? = 1 ]]; then + echo "Press any key to return menu!" + read + fi + ;; + "a") + exit_install + ;; + "q") + exit_install + ;; + *) + menu_index="0" + menu_choice="0" + menu + ;; + esac +done diff --git a/dev-support/submarine-installer/package/calico/calico-node.service b/dev-support/submarine-installer/package/calico/calico-node.service new file mode 100644 index 0000000000000000000000000000000000000000..744c4191f9405ce596d87f50b7c0b92f9ff1ba87 --- /dev/null +++ b/dev-support/submarine-installer/package/calico/calico-node.service @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=calico node +After=docker.service +Requires=docker.service + +[Service] +User=root +PermissionsStartOnly=true +ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \ + -e ETCD_ENDPOINTS=ETCD_ENDPOINTS_REPLACE \ + -e CALICO_LIBNETWORK_ENABLED=true \ + -e CALICO_NETWORKING_BACKEND=bird \ + -e CALICO_DISABLE_FILE_LOGGING=true \ + -e CALICO_IPV4POOL_CIDR=CALICO_IPV4POOL_CIDR_REPLACE/16 \ + -e CALICO_IPV4POOL_IPIP=always \ + -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \ + -e FELIX_IPV6SUPPORT=false \ + -e FELIX_LOGSEVERITYSCREEN=info \ + -e FELIX_IPINIPMTU=1440 \ + -e FELIX_HEALTHENABLED=true \ + -e IP= \ + -v /etc/calico/ssl:/etc/calico/ssl \ + -v /var/run/calico:/var/run/calico \ + -v /lib/modules:/lib/modules \ + -v /run/docker/plugins:/run/docker/plugins \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/log/calico:/var/log/calico \ + calico/node:v2.6.2 +ExecStop=/usr/bin/docker rm -f calico-node +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/dev-support/submarine-installer/package/calico/calicoctl.cfg b/dev-support/submarine-installer/package/calico/calicoctl.cfg new file mode 100644 index 0000000000000000000000000000000000000000..44ada867a6265cad2ca1167fbd379e590f480a17 --- /dev/null +++ b/dev-support/submarine-installer/package/calico/calicoctl.cfg @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: calicoApiConfig +metadata: +spec: + datastoreType: "etcdv2" + etcdEndpoints: ETCD_ENDPOINTS_REPLACE \ No newline at end of file diff --git a/dev-support/submarine-installer/package/docker/daemon.json b/dev-support/submarine-installer/package/docker/daemon.json new file mode 100644 index 0000000000000000000000000000000000000000..5e7e0cf2b379c5fbfebd49d4ec9d3227096bdad6 --- /dev/null +++ b/dev-support/submarine-installer/package/docker/daemon.json @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "insecure-registries": ["DOCKER_REGISTRY_REPLACE"], + "cluster-store":"CLUSTER_STORE_REPLACE", + "cluster-advertise":"LOCAL_HOST_IP_REPLACE:2375", + "dns": ["YARN_REGISTRY_DNS_HOST_REPLACE", LOCAL_DNS_HOST_REPLACE], + "hosts": ["tcp://LOCAL_HOST_IP_REPLACE:2375","unix:///var/run/docker.sock"] +} diff --git a/dev-support/submarine-installer/package/docker/docker.service b/dev-support/submarine-installer/package/docker/docker.service new file mode 100644 index 0000000000000000000000000000000000000000..f444f995d5791cd16ae78bd0afd78881d399c823 --- /dev/null +++ b/dev-support/submarine-installer/package/docker/docker.service @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.io + +[Service] +Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin" +ExecStart=/usr/bin/dockerd --log-level=error +ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT +ExecReload=/bin/kill -s HUP $MAINPID +Restart=on-failure +RestartSec=5 +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/dev-support/submarine-installer/package/etcd/etcd.service b/dev-support/submarine-installer/package/etcd/etcd.service new file mode 100644 index 0000000000000000000000000000000000000000..5a44ce2cfa31434a0eca9e1cc4019638a0a61b7f --- /dev/null +++ b/dev-support/submarine-installer/package/etcd/etcd.service @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=etcd +Documentation=https://github.com/coreos/etcd + +[Service] +Type=notify +Restart=always +RestartSec=5s +LimitNOFILE=40000 +TimeoutStartSec=0 + +ExecStart=/usr/bin/etcd \ + --name=ETCD_NODE_NAME_REPLACE \ + --data-dir=/var/lib/etcd \ + --listen-client-urls=http://LOCAL_HOST_REPLACE:2379,http://127.0.0.1:2379 \ + --listen-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ + --advertise-client-urls=http://LOCAL_HOST_REPLACE:2379 \ + --initial-advertise-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ + --initial-cluster=INITIAL_CLUSTER_REPLACE \ + --initial-cluster-token=etcd-token \ + --initial-cluster-state=new + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc b/dev-support/submarine-installer/package/hadoop/yarn/etc new file mode 120000 index 0000000000000000000000000000000000000000..c33e2a06e834e51813321e01fa56b740a19994b2 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc @@ -0,0 +1 @@ +etc_secure/ \ No newline at end of file diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/capacity-scheduler.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/capacity-scheduler.xml new file mode 100644 index 0000000000000000000000000000000000000000..742fcd2fecc6d874e6d30369abb1e7c4290f16b2 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/capacity-scheduler.xml @@ -0,0 +1,131 @@ + + + + + yarn.scheduler.capacity.maximum-applications + + 20000 + + Maximum number of applications that can be pending and running. + + + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.2 + + Maximum percent of resources in the cluster which can be used to run + application masters i.e. controls number of concurrent running + applications. + + + + + yarn.scheduler.capacity.resource-calculator + + org.apache.hadoop.yarn.util.resource.DominantResourceCalculator + + The ResourceCalculator implementation to be used to compare + Resources in the scheduler. + The default i.e. DefaultResourceCalculator only uses Memory while + DominantResourceCalculator uses dominant-resource to compare + multi-dimensional resources such as Memory, CPU etc. + + + + + yarn.scheduler.capacity.root.queues + default,dev + + The queues at the this level (root is the root queue). + + + + + yarn.scheduler.capacity.root.default.capacity + [memory=100Gi,vcores=20,gpu=1] + Default queue target capacity. + + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + The default value is 1 + + + + + yarn.scheduler.capacity.root.default.maximum-capacity + [memory=200Gi,vcores=40,gpu=2] + + The maximum capacity of the default queue. + + + + + yarn.scheduler.capacity.root.default.user-limit-factor + 2.0 + + + + yarn.scheduler.capacity.root.dev.capacity + [memory=100Gi,vcores=20,gpu=1] + Default queue target capacity. + + + + yarn.scheduler.capacity.root.dev.maximum-capacity + [memory=200Gi,vcores=40,gpu=2] + + The maximum capacity of the default queue. + + + + yarn.scheduler.capacity.root.default.state + RUNNING + + The state of the default queue. State can be one of RUNNING or STOPPED. + + + + + yarn.scheduler.capacity.root.default.acl_submit_applications + * + + The ACL of who can submit jobs to the default queue. + + + + + yarn.scheduler.capacity.root.default.acl_administer_queue + * + + The ACL of who can administer jobs on the default queue. + + + + + yarn.scheduler.capacity.node-locality-delay + 40 + + Number of missed scheduling opportunities after which the CapacityScheduler + attempts to schedule rack-local containers. + Typically this should be set to number of nodes in the cluster, By default is setting + approximately number of nodes in one rack which is 40. + + + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/container-executor.cfg b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/container-executor.cfg new file mode 100644 index 0000000000000000000000000000000000000000..40219eccce973539bb53a992a862a9fa905a40ef --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/container-executor.cfg @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE +yarn.nodemanager.linux-container-executor.group=yarn +yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE +banned.users=root +allowed.system.users=yarn +min.user.id=500 + +[docker] +module.enabled=true +docker.binary=/usr/bin/docker +docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN +docker.allowed.networks=bridge,host,none,CALICO_NETWORK_NAME_REPLACE +docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,regex:^nvidia_driver_.*$ +docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE +docker.privileged-containers.enabled=false +docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE +docker.allowed.volume-drivers=nvidia-docker +docker.allowed.devices=regex:^/dev/nvidia.*$ +docker.allowed.runtimes=nvidia + +[gpu] +module.enabled=true + +[cgroups] +root=/sys/fs/cgroup +yarn-hierarchy=YARN_HIERARCHY_REPLACE diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/core-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/core-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..72c7f2ed2e711bf7ebad81ad4e94ddce857dd816 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/core-site.xml @@ -0,0 +1,209 @@ + + + + + + + + + fs.defaultFS + FS_DEFAULTFS_REPLACE + The name of the default file system. Either the literal string "local" or a host:port for NDFS. + true + + + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.Lz4Codec,org.apache.hadoop.io.compress.SnappyCodec + + + + io.compression.codec.lzo.class + com.hadoop.compression.lzo.LzoCodec + + + + + webinterface.private.actions + false + If set to true, the web interfaces of JT and NN may + contain + actions, such as kill job, delete file, etc., that should + not be exposed to public. Enable this option if the interfaces + are only reachable by those who have the right authorization. + + + + + hadoop.security.authentication + simple + + Set the authentication for the cluster. Valid values are: simple or + kerberos. + + + + + hadoop.security.authorization + false + + Enable authorization for different protocols. + + + + + hadoop.security.groups.cache.secs + 14400 + + + + + hadoop.http.filter.initializers + org.apache.hadoop.security.AuthenticationFilterInitializer + + + hadoop.http.authentication.type + simple + + + + hadoop.http.authentication.token.validity + 36000 + + + hadoop.http.authentication.signature.secret.file + HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE_REPLACE + + The signature secret for signing the authentication tokens. + If not set a random secret is generated at startup time. + The same secret should be used for JT/NN/DN/TT configurations. + + + + + hadoop.http.authentication.simple.anonymous.allowed + true + + + + hadoop.tmp.dir + /tmp/hadoop-${user.name} + + + + io.bytes.per.checksum + 4096 + + + + fs.inmemory.size.mb + 200 + + + + io.file.buffer.size + 131072 + + + + hadoop.proxyuser.mapred.hosts + * + + + hadoop.proxyuser.mapred.groups + * + + + + hadoop.proxyuser.hive.hosts + * + + + hadoop.proxyuser.hive.groups + * + + + + hadoop.proxyuser.hadoop.hosts + * + + + hadoop.proxyuser.hadoop.groups + * + + + + hadoop.proxyuser.yarn.hosts + * + + + hadoop.proxyuser.yarn.groups + * + + + + + + Is the registry enabled in the YARN Resource Manager? + If true, the YARN RM will, as needed. + create the user and system paths, and purge + service records when containers, application attempts + and applications complete. + If false, the paths must be created by other means, + and no automatic cleanup of service records will take place. + + hadoop.registry.rm.enabled + false + + + + + A comma separated list of hostname:port pairs defining the + zookeeper quorum binding for the registry + + hadoop.registry.zk.quorum + YARN_ZK_ADDRESS_REPLACE + + + + + The root zookeeper node for the registry + + hadoop.registry.zk.root + /registry + + + + + Zookeeper connection retry count before failing + + hadoop.registry.zk.retry.times + 5 + + + + + + hadoop.registry.zk.retry.interval.ms + 1000 + + + + + Zookeeper session timeout in milliseconds + + hadoop.registry.zk.connection.timeout.ms + 15000 + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/gpu/yarn-site-gpu.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/gpu/yarn-site-gpu.xml new file mode 100644 index 0000000000000000000000000000000000000000..f76c9b6460fdfb6f44cec56db1af0a6a698ba6f4 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/gpu/yarn-site-gpu.xml @@ -0,0 +1,27 @@ + + + + + + + + yarn.nodemanager.resource-plugins + yarn.io/gpu + + + + yarn.nodemanager.resource-plugins.gpu.docker-plugin + nvidia-docker-v2 + + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/hadoop-env.sh b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/hadoop-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..260b9e451ddbfa9541c1f73da64b2b03aa1928b7 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/hadoop-env.sh @@ -0,0 +1,108 @@ +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. +#export JAVA_HOME=${JAVA_HOME} + + +###mod +export JAVA_HOME="JAVA_HOME_REPLACE" +export HADOOP_HOME="HADOOP_HOME_REPLACE" + +#export HADOOP_ROOT_LOGGER=INFO,DRFA +export HADOOP_SECURITY_LOGGER=INFO,DRFAS +export HDFS_AUDIT_LOGGER=INFO,DRFAAUDIT + +export JSVC_HOME=$HADOOP_HOME/bin +export HADOOP_CLASSPATH=$HADOOP_HOME/lib/native:$HADOOP_CLASSPATH +# default heapsize for hadoop daemon +export HADOOP_HEAPSIZE=5120 +export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" +export HADOOP_LOG_DIR="YARN_LOG_DIR_REPLACE" +export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_LOG_DIR +export HADOOP_PID_DIR="YARN_PID_DIR_REPLACE" +export HADOOP_SECURE_DN_PID_DIR=$HADOOP_PID_DIR + +export HADOOP_NAMENODE_USER=hdfs +export HADOOP_DATANODE_USER=hdfs +export HADOOP_SECURE_DN_USER=hdfs + +GC_LOG_DATE=`date +%F` +GC_LOG_DIR="GC_LOG_DIR_REPLACE" +export NAMENODE_JVM_OPTS="-Dcom.netease.appname=NameNode -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/namenode.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx71680m -Xms71680m -Xmn20480m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +export DATANODE_JVM_OPTS="-Dcom.netease.appname=DataNode -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/datanode.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx10240m -Xms10240m -Xmn2048m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +###mod + + +# The jsvc implementation to use. Jsvc is required to run secure datanodes. +#export JSVC_HOME=${JSVC_HOME} + +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"} + +# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler. +for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do + if [ "$HADOOP_CLASSPATH" ]; then + export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f + else + export HADOOP_CLASSPATH=$f + fi +done + +# The maximum amount of heap to use, in MB. Default is 1000. +#export HADOOP_HEAPSIZE= +#export HADOOP_NAMENODE_INIT_HEAPSIZE="" + +# Extra Java runtime options. Empty by default. +# export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" + +# Command specific options appended to HADOOP_OPTS when specified +export HADOOP_NAMENODE_OPTS="$NAMENODE_JVM_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS" +export HADOOP_DATANODE_OPTS="$DATANODE_JVM_OPTS -Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS" + +export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +#export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS" +#export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=6002" +#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS" + +# On secure datanodes, user to run the datanode as after dropping privileges +#export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} + +# Where log files are stored. $HADOOP_HOME/logs by default. +#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER + +# Where log files are stored in the secure data environment. +#export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} + +# The directory where pid files are stored. /tmp by default. +# NOTE: this should be set to a directory that can only be written to by +# the user that will run the hadoop daemons. Otherwise there is the +# potential for a symlink attack. +#export HADOOP_PID_DIR=${HADOOP_PID_DIR} +#export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR} + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/hdfs-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/hdfs-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..e5dfce6fac61d79fa32ab70fc2d57c424afe400a --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/hdfs-site.xml @@ -0,0 +1,79 @@ + + + + + + + local.hostname.nn1 + HDFS_NAMENODE_HOSTS1_REPLACE + + + local.hostname.nn2 + HDFS_NAMENODE_HOSTS2_REPLACE + + + + dfs.ha.namenodes.LOCAL_REALM_REPLACE + nn1,nn2 + + + + dfs.namenode.rpc-address.LOCAL_REALM_REPLACE.nn1 + ${local.hostname.nn1}:8020 + + + dfs.namenode.rpc-address.LOCAL_REALM_REPLACE.nn2 + ${local.hostname.nn2}:8020 + + + + dfs.nameservices + LOCAL_REALM_REPLACE + + + + dfs.client.failover.proxy.provider.LOCAL_REALM_REPLACE + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + + + fs.permissions.umask-mode + 027 + + + + dfs.blocksize + 268435456 + + + + dfs.client.read.shortcircuit + true + + + + dfs.client.read.shortcircuit.streams.cache.size + 1024 + + + + dfs.client.read.shortcircuit.streams.cache.size.expiry.ms + 300000 + + + + dfs.domain.socket.path + /var/lib/hadoop-hdfs/dn_socket + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/mapred-env.sh b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/mapred-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..84bee0b3d3fd300bce995d53df442dce7c2cb8a1 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/mapred-env.sh @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +## THIS FILE ACTS AS AN OVERRIDE FOR hadoop-env.sh FOR ALL +## WORK DONE BY THE mapred AND RELATED COMMANDS. +## +## Precedence rules: +## +## mapred-env.sh > hadoop-env.sh > hard-coded defaults +## +## MAPRED_xyz > HADOOP_xyz > hard-coded defaults +## + +### +# Job History Server specific parameters +### + +# Specify the max heapsize for the JobHistoryServer. If no units are +# given, it will be assumed to be in MB. +# This value will be overridden by an Xmx setting specified in HADOOP_OPTS, +# and/or MAPRED_HISTORYSERVER_OPTS. +# Default is the same as HADOOP_HEAPSIZE_MAX. +#export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=5120 + +# Specify the JVM options to be used when starting the HistoryServer. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +GC_LOG_DATE=`date +%F` +GC_LOG_DIR="GC_LOG_DIR_REPLACE" +export MAPRED_HISTORYSERVER_OPTS="-server -Xmx5g -Xms5g -Xmn2g -XX:SurvivorRatio=4 -Xss256k -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -Xloggc:${GC_LOG_DIR}/rm.gc.log-${GC_LOG_DATE}" + +# Specify the log4j settings for the JobHistoryServer +# Java property: hadoop.root.logger +#export HADOOP_JHS_LOGGER=INFO,RFA diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/mapred-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/mapred-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..19c74336194601f415ad1f113c531302af6abe99 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/mapred-site.xml @@ -0,0 +1,120 @@ + + + + + + + mapreduce.framework.name + yarn + + + + yarn.app.mapreduce.am.staging-dir + YARN_APP_MAPREDUCE_AM_STAGING_DIR_REPLACE + + + + + mapreduce.jobhistory.address + YARN_JOB_HISTORY_HOST_REPLACE:10020 + + + mapreduce.jobhistory.webapp.address + YARN_JOB_HISTORY_HOST_REPLACE:19888 + + + + + yarn.app.mapreduce.am.resource.mb + 1536 + + + mapreduce.map.memory.mb + 1024 + + + mapreduce.reduce.memory.mb + 1536 + + + mapreduce.map.java.opts + -Xmx820M + + + + mapreduce.reduce.java.opts + -Xmx1228M + + + yarn.app.mapreduce.am.command-opts + -Xmx1228M + + + + + + io.sort.factor + 100 + + + io.sort.mb + 200 + + + + + mapreduce.map.speculative + false + + + mapreduce.reduce.speculative + false + + + + + mapreduce.map.output.compress + true + + + mapreduce.map.output.compress.codec + com.hadoop.compression.lzo.LzoCodec + + + + yarn.app.mapreduce.am.env + HADOOP_MAPRED_HOME=$HADOOP_HOME + + + mapreduce.map.env + HADOOP_MAPRED_HOME=$HADOOP_HOME + + + mapreduce.reduce.env + HADOOP_MAPRED_HOME=$HADOOP_HOME + + + + mapreduce.job.hdfs-servers + FS_DEFAULTFS_REPLACE + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/resource-types.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/resource-types.xml new file mode 100755 index 0000000000000000000000000000000000000000..54ea7dcf6d678fd47e45ab86ab5a4a00c2c65eb3 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/resource-types.xml @@ -0,0 +1,6 @@ + + + yarn.resource-types + yarn.io/gpu + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/yarn-env.sh b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/yarn-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..41c27e15110af913dc2cba9354a335731d5e3a07 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/yarn-env.sh @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +###mod +export YARN_ROOT_LOGGER=INFO,DRFA +#export YARN_ROOT_LOGGER=DEBUG,DRFA +export YARN_LOG_DIR=$HADOOP_LOG_DIR +export YARN_PID_DIR=$HADOOP_PID_DIR + +GC_LOG_DATE=`date +%F` +GC_LOG_DIR="GC_LOG_DIR_REPLACE" +#export RESOURCEMANAGER_JVM_OPTS="-Dcom.netease.appname=ResourceManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/resourcemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx20480m -Xms20480m -Xmn5120m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +export RESOURCEMANAGER_JVM_OPTS="-Dcom.netease.appname=ResourceManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/resourcemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=6001" +export NODEMANAGER_JVM_OPTS="-Dcom.netease.appname=NodeManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/nodemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx5120m -Xms5120m -Xmn1024m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +#export NODEMANAGER_JVM_OPTS="-Dcom.netease.appname=NodeManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/nodemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=6001" +###mod +#export YARN_TIMELINESERVER_OPTS="-Dcom.netease.appname=TimelineServer -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/timelineserver.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=6001" +#export YARN_TIMELINEREADER_OPTS="-Dcom.netease.appname=TimelineReaderServer -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/timelineserver.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=6001" + +# User for YARN daemons +export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} + +# resolve links - $0 may be a softlink +export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}" + +# some Java parameters +# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi + +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# For setting YARN specific HEAP sizes please use this +# Parameter and set appropriately +# YARN_HEAPSIZE=1000 + +# check envvars which might override default args +if [ "$YARN_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" +fi + +# Resource Manager specific parameters + +# Specify the max Heapsize for the ResourceManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_RESOURCEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +#export YARN_RESOURCEMANAGER_HEAPSIZE=1000 + +# Specify the JVM options to be used when starting the ResourceManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +export YARN_RESOURCEMANAGER_OPTS="$RESOURCEMANAGER_JVM_OPTS" + +# Node Manager specific parameters + +# Specify the max Heapsize for the NodeManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_NODEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +#export YARN_NODEMANAGER_HEAPSIZE=1000 + +# Specify the JVM options to be used when starting the NodeManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +export YARN_NODEMANAGER_OPTS="$NODEMANAGER_JVM_OPTS" + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + + +# default log directory & file +if [ "$YARN_LOG_DIR" = "" ]; then + YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" +fi +if [ "$YARN_LOGFILE" = "" ]; then + YARN_LOGFILE='yarn.log' +fi + +# default policy file for service-level authorization +if [ "$YARN_POLICYFILE" = "" ]; then + YARN_POLICYFILE="hadoop-policy.xml" +fi + +# restore ordinary behaviour +unset IFS + + +YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" +YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" +YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" +fi +YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/yarn-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/yarn-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..ff0fc17cf0a136b99cd21474b75dc80a0aadeac5 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_insecure/hadoop/yarn-site.xml @@ -0,0 +1,655 @@ + + + + + + + local.hostname.rm1 + YARN_RESOURCE_MANAGER_HOSTS1_REPLACE + + + local.hostname.rm2 + YARN_RESOURCE_MANAGER_HOSTS2_REPLACE + + + local.cluster-id + LOCAL_CLUSTER_ID_REPLACE + + + + + yarn.resourcemanager.zk-address + YARN_ZK_ADDRESS_REPLACE + + + yarn.resourcemanager.recovery.enabled + true + + + yarn.resourcemanager.store.class + org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore + + + yarn.resourcemanager.zk-timeout-ms + 90000 + + + yarn.resourcemanager.am.max-attempts + 3 + + + yarn.resourcemanager.state-store.max-completed-applications + 200 + + + + + yarn.resourcemanager.ha.enabled + true + + + yarn.resourcemanager.cluster-id + ${local.cluster-id} + + + yarn.resourcemanager.ha.rm-ids + rm1,rm2 + + + + yarn.resourcemanager.hostname.rm1 + ${local.hostname.rm1} + + + + yarn.resourcemanager.hostname.rm2 + ${local.hostname.rm2} + + + + yarn.resourcemanager.webapp.address.rm1 + ${local.hostname.rm1}:8088 + + + + yarn.resourcemanager.webapp.address.rm2 + ${local.hostname.rm2}:8088 + + + + + yarn.nodemanager.aux-services + mapreduce_shuffle,spark_shuffle,timeline_collector + + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + yarn.nodemanager.aux-services.spark_shuffle.class + org.apache.spark.network.yarn.YarnShuffleService + + + spark.yarn.shuffle.stopOnFailure + true + + + yarn.nodemanager.aux-services.timeline_collector.class + org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService + + + + + yarn.nodemanager.local-dirs + YARN_NODEMANAGER_LOCAL_DIRS_REPLACE + + + yarn.nodemanager.log-dirs + YARN_NODEMANAGER_LOG_DIRS_REPLACE + + + + + Where to aggregate logs in hdfs + yarn.nodemanager.remote-app-log-dir + YARN_AGGREGATED_LOG_DIR_REPLACE + + + yarn.log-aggregation-enable + true + + + yarn.log-aggregation.retain-seconds + 604800 + + + yarn.log-aggregation.retain-check-interval-seconds + 86400 + + + yarn.nodemanager.log-aggregation.compression-type + gz + + + + + + + + + yarn.nodemanager.container-executor.class + org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor + + + yarn.nodemanager.linux-container-executor.path + /etc/yarn/sbin/Linux-amd64-64/container-executor + + + yarn.nodemanager.linux-container-executor.group + yarn + + + + + Miniumum request grant-able by the RM scheduler + yarn.scheduler.minimum-allocation-mb + 512 + + + Maximum request grant-able by the RM scheduler + yarn.scheduler.maximum-allocation-mb + 32768 + + + yarn.scheduler.minimum-allocation-vcores + 1 + + + yarn.scheduler.maximum-allocation-vcores + 30 + + + yarn.resourcemanager.scheduler.class + + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + + yarn.scheduler.increment-allocation-mb + 512 + + + yarn.scheduler.fair.allow-undeclared-pools + false + + + + yarn.client.nodemanager-connect.max-wait-ms + 60000 + + + + + yarn.resourcemanager.client.thread-count + 64 + + + yarn.resourcemanager.resource-tracker.client.thread-count + 64 + + + yarn.resourcemanager.scheduler.client.thread-count + 64 + + + yarn.nodemanager.container-manager.thread-count + 32 + + + yarn.nodemanager.container-metrics.enable + true + Todo hbase metrics cleanup mechanism + + + + + The minimum space that must be available on a disk for + it to be used. This applies to yarn.nodemanager.local-dirs and + yarn.nodemanager.log-dirs. + yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb + 1000 + + + Defines how often NMs wake up to upload log files. + The default value is -1. By default, the logs will be uploaded when + the application is finished. By setting this configure, logs can be uploaded + periodically when the application is running. The minimum rolling-interval-seconds + can be set is 3600. + + yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds + 3600 + + + Enable the node manager to recover after starting + yarn.nodemanager.recovery.enabled + true + + + The local filesystem directory in which the node manager will + store state when recovery is enabled. + yarn.nodemanager.recovery.dir + YARN_NODEMANAGER_RECOVERY_DIR_REPLACE + + + + + yarn.nodemanager.resource.memory-mb + 204800 + + + yarn.nodemanager.resource.cpu-vcores + 40 + + + + yarn.admin.acl + hadoop,yarn + + + yarn.acl.enable + true + + + + + yarn.resourcemanager.nodes.exclude-path + YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH_REPLACE + + + If true, ResourceManager will have proxy-user privileges. + Use case: In a secure cluster, YARN requires the user hdfs delegation-tokens to + do localization and log-aggregation on behalf of the user. If this is set to true, + ResourceManager is able to request new hdfs delegation tokens on behalf of + the user. This is needed by long-running-service, because the hdfs tokens + will eventually expire and YARN requires new valid tokens to do localization + and log-aggregation. Note that to enable this use case, the corresponding + HDFS NameNode has to configure ResourceManager as the proxy-user so that + ResourceManager can itself ask for new tokens on behalf of the user when + tokens are past their max-life-time. + yarn.resourcemanager.proxy-user-privileges.enabled + true + + + + + yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled + false + + + + hadoop.http.filter.initializers + org.apache.hadoop.security.AuthenticationFilterInitializer + + + hadoop.http.authentication.type + true + + + + + yarn.nodemanager.container.use.serialgc.enable + true + + + + + Indicate to clients whether Timeline service is enabled or not. + If enabled, the TimelineClient library used by end-users will post entities + and events to the Timeline server. + yarn.timeline-service.enabled + true + + + + The setting that controls whether yarn system metrics is + published on the timeline server or not by RM. + yarn.resourcemanager.system-metrics-publisher.enabled + true + + + + Indicate to clients whether to query generic application + data from timeline history-service or not. If not enabled then application + data is queried only from Resource Manager. + yarn.timeline-service.generic-application-history.enabled + true + + + + yarn.timeline-service.http-authentication.type + simple + + + + + yarn.timeline-service.hostname + YARN_TIMELINE_HOST_REPLACE + + + + yarn.timeline-service.hbase.coprocessor.jar.hdfs.location + /hbase/coprocessor + + + + URL for log aggregation server web service + yarn.log.server.web-service.url + http://YARN_TIMELINE_HOST_REPLACE:8189/ws/v2/applicationlog + + + + + yarn.timeline-service.leveldb-timeline-store.path + YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_REPLACE + + + + yarn.timeline-service.leveldb-state-store.path + YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_REPLACE + + + + yarn.timeline-service.leveldb-timeline-store.read-cache-size + 104857600 + + + + yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size + 10000 + + + + yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size + 10000 + + + + yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms + 300000 + + + + yarn.timeline-service.recovery.enabled + true + + + + yarn.timeline-service.state-store-class + org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore + + + + yarn.timeline-service.store-class + org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore + + + + yarn.timeline-service.ttl-enable + true + + + + yarn.timeline-service.ttl-ms + 2678400000 + + + + + yarn.timeline-service.version + 2.0f + + + + The setting that controls whether yarn system metrics is + published on the Timeline service or not by RM And NM. + yarn.system-metrics-publisher.enabled + true + + + + The setting that controls whether yarn container events are + published to the timeline service or not by RM. This configuration setting + is for ATS V2. + + yarn.rm.system-metrics-publisher.emit-container-events + true + + + + Optional URL to an hbase-site.xml configuration file to be + used to connect to the timeline-service hbase cluster. If empty or not + specified, then the HBase configuration will be loaded from the classpath. + When specified the values in the specified configuration file will override + those from the ones that are present on the classpath. + + yarn.timeline-service.hbase.configuration.file + YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE_REPLACE + + + + + yarn.timeline-service.versions + 1.5f,2.0f + + + + yarn.timeline-service.entity-group-fs-store.active-dir + YARN_TIMELINE_FS_STORE_DIR_REPLACE/active + + + + yarn.timeline-service.entity-group-fs-store.done-dir + YARN_TIMELINE_FS_STORE_DIR_REPLACE/done + + + + + yarn.timeline-service.reader.webapp.address + YARN_TIMELINE_HOST_REPLACE:8189 + + + + yarn.timeline-service.entity-group-fs-store.summary-store + org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore + + + + Enable services rest api on ResourceManager. + yarn.webapp.api-service.enable + true + + + + The domain name for Hadoop cluster associated records. As short as possible. + hadoop.registry.dns.domain-name + ml + + + + The DNS functionality is enabled for the cluster. Default is false. + hadoop.registry.dns.enabled + true + + + Address associated with the network interface to which the DNS listener should bind. + hadoop.registry.dns.bind-address + YARN_REGISTRY_DNS_HOST_REPLACE + + + The port number for the DNS listener. The default port is 5353. + If the standard privileged port 53 is used, make sure start the DNS with jsvc support. + hadoop.registry.dns.bind-port + + YARN_REGISTRY_DNS_HOST_PORT_REPLACE + + + + + yarn.nodemanager.linux-container-executor.resources-handler.class + org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler + + + yarn.nodemanager.linux-container-executor.cgroups.mount + false + + + yarn.nodemanager.resource.percentage-physical-cpu-limit + 80 + + + Whether virtual memory limits will be enforced for containers. + yarn.nodemanager.vmem-check-enabled + false + + + + Whether YARN CGroups memory tracking is enabled. + yarn.nodemanager.resource.memory.enabled + true + + + + + yarn.nodemanager.runtime.linux.docker.default-container-network + + CALICO_NETWORK_NAME_REPLACE + + + yarn.nodemanager.runtime.linux.allowed-runtimes + default,docker + + + yarn.nodemanager.runtime.linux.docker.allowed-container-networks + host,none,bridge,CALICO_NETWORK_NAME_REPLACE + + + yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed + false + + + yarn.nodemanager.runtime.linux.docker.privileged-containers.acl + + + + yarn.nodemanager.runtime.linux.docker.capabilities + CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE + + + + + yarn.webapp.ui2.enable + true + + + + + + yarn.service.base.path + /tmp/.LOCAL_CLUSTER_ID_REPLACE/ + + + + yarn.nodemanager.address + 0.0.0.0:45454 + + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/capacity-scheduler.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/capacity-scheduler.xml new file mode 100644 index 0000000000000000000000000000000000000000..742fcd2fecc6d874e6d30369abb1e7c4290f16b2 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/capacity-scheduler.xml @@ -0,0 +1,131 @@ + + + + + yarn.scheduler.capacity.maximum-applications + + 20000 + + Maximum number of applications that can be pending and running. + + + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.2 + + Maximum percent of resources in the cluster which can be used to run + application masters i.e. controls number of concurrent running + applications. + + + + + yarn.scheduler.capacity.resource-calculator + + org.apache.hadoop.yarn.util.resource.DominantResourceCalculator + + The ResourceCalculator implementation to be used to compare + Resources in the scheduler. + The default i.e. DefaultResourceCalculator only uses Memory while + DominantResourceCalculator uses dominant-resource to compare + multi-dimensional resources such as Memory, CPU etc. + + + + + yarn.scheduler.capacity.root.queues + default,dev + + The queues at the this level (root is the root queue). + + + + + yarn.scheduler.capacity.root.default.capacity + [memory=100Gi,vcores=20,gpu=1] + Default queue target capacity. + + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + The default value is 1 + + + + + yarn.scheduler.capacity.root.default.maximum-capacity + [memory=200Gi,vcores=40,gpu=2] + + The maximum capacity of the default queue. + + + + + yarn.scheduler.capacity.root.default.user-limit-factor + 2.0 + + + + yarn.scheduler.capacity.root.dev.capacity + [memory=100Gi,vcores=20,gpu=1] + Default queue target capacity. + + + + yarn.scheduler.capacity.root.dev.maximum-capacity + [memory=200Gi,vcores=40,gpu=2] + + The maximum capacity of the default queue. + + + + yarn.scheduler.capacity.root.default.state + RUNNING + + The state of the default queue. State can be one of RUNNING or STOPPED. + + + + + yarn.scheduler.capacity.root.default.acl_submit_applications + * + + The ACL of who can submit jobs to the default queue. + + + + + yarn.scheduler.capacity.root.default.acl_administer_queue + * + + The ACL of who can administer jobs on the default queue. + + + + + yarn.scheduler.capacity.node-locality-delay + 40 + + Number of missed scheduling opportunities after which the CapacityScheduler + attempts to schedule rack-local containers. + Typically this should be set to number of nodes in the cluster, By default is setting + approximately number of nodes in one rack which is 40. + + + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/container-executor.cfg b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/container-executor.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c4bb5ea8493b360ac4145dda3eb93576881be34d --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/container-executor.cfg @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE +yarn.nodemanager.linux-container-executor.group=yarn +yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE +banned.users=root +allowed.system.users=yarn +min.user.id=500 + +[docker] +module.enabled=true +docker.binary=/usr/bin/docker +docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN +docker.allowed.networks=YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS_REPLACE +docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,HADOOP_HOME_REPLACE,regex:^nvidia_driver_.*$ +docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE +docker.privileged-containers.enabled=false +docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE +docker.allowed.volume-drivers=nvidia-docker +docker.allowed.devices=regex:^/dev/nvidia.*$ +docker.allowed.runtimes=nvidia + +[gpu] +module.enabled=true + +[cgroups] +root=/sys/fs/cgroup +yarn-hierarchy=YARN_HIERARCHY_REPLACE diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/core-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/core-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..27ed6419e08c24008a40ce3331c2a4cbf7ea51d1 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/core-site.xml @@ -0,0 +1,235 @@ + + + + + + + local.realm + LOCAL_REALM_REPLACE + + + + + fs.defaultFS + FS_DEFAULTFS_REPLACE + The name of the default file system. Either the literal string "local" or a host:port for NDFS. + true + + + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.Lz4Codec,org.apache.hadoop.io.compress.SnappyCodec + + + + io.compression.codec.lzo.class + com.hadoop.compression.lzo.LzoCodec + + + + + webinterface.private.actions + false + If set to true, the web interfaces of JT and NN may + contain + actions, such as kill job, delete file, etc., that should + not be exposed to public. Enable this option if the interfaces + are only reachable by those who have the right authorization. + + + + + hadoop.security.authentication + kerberos + + Set the authentication for the cluster. Valid values are: simple or + kerberos. + + + + + hadoop.security.authorization + true + + Enable authorization for different protocols. + + + + + hadoop.security.groups.cache.secs + 14400 + + + + hadoop.kerberos.kinit.command + /usr/bin/kinit + + + + + hadoop.http.filter.initializers + org.apache.hadoop.security.AuthenticationFilterInitializer + + + + hadoop.http.authentication.type + kerberos + + + + hadoop.http.authentication.token.validity + 36000 + + + + hadoop.http.authentication.signature.secret.file + HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE_REPLACE + + The signature secret for signing the authentication tokens. + If not set a random secret is generated at startup time. + The same secret should be used for JT/NN/DN/TT configurations. + + + + + hadoop.http.authentication.cookie.domain + ${local.realm} + + + + hadoop.http.authentication.simple.anonymous.allowed + true + + + + hadoop.http.authentication.kerberos.principal + HTTP/_HOST@${local.realm} + + + + hadoop.http.authentication.kerberos.keytab + HTTP_KEYTAB_LOCATION_REPLACE + + + + hadoop.tmp.dir + /tmp/hadoop-${user.name} + + + + io.bytes.per.checksum + 4096 + + + + fs.inmemory.size.mb + 200 + + + + io.file.buffer.size + 131072 + + + + hadoop.proxyuser.mapred.hosts + * + + + hadoop.proxyuser.mapred.groups + * + + + + hadoop.proxyuser.hive.hosts + * + + + hadoop.proxyuser.hive.groups + * + + + + hadoop.proxyuser.hadoop.hosts + * + + + hadoop.proxyuser.hadoop.groups + * + + + + hadoop.proxyuser.yarn.hosts + * + + + hadoop.proxyuser.yarn.groups + * + + + + + + Is the registry enabled in the YARN Resource Manager? + If true, the YARN RM will, as needed. + create the user and system paths, and purge + service records when containers, application attempts + and applications complete. + If false, the paths must be created by other means, + and no automatic cleanup of service records will take place. + + hadoop.registry.rm.enabled + false + + + + + A comma separated list of hostname:port pairs defining the + zookeeper quorum binding for the registry + + hadoop.registry.zk.quorum + YARN_ZK_ADDRESS_REPLACE + + + + + The root zookeeper node for the registry + + hadoop.registry.zk.root + /registry + + + + + Zookeeper connection retry count before failing + + hadoop.registry.zk.retry.times + 5 + + + + + + hadoop.registry.zk.retry.interval.ms + 1000 + + + + + Zookeeper session timeout in milliseconds + + hadoop.registry.zk.connection.timeout.ms + 15000 + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/gpu/yarn-site-gpu.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/gpu/yarn-site-gpu.xml new file mode 100644 index 0000000000000000000000000000000000000000..f76c9b6460fdfb6f44cec56db1af0a6a698ba6f4 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/gpu/yarn-site-gpu.xml @@ -0,0 +1,27 @@ + + + + + + + + yarn.nodemanager.resource-plugins + yarn.io/gpu + + + + yarn.nodemanager.resource-plugins.gpu.docker-plugin + nvidia-docker-v2 + + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/hadoop-env.sh b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/hadoop-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..260b9e451ddbfa9541c1f73da64b2b03aa1928b7 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/hadoop-env.sh @@ -0,0 +1,108 @@ +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. +#export JAVA_HOME=${JAVA_HOME} + + +###mod +export JAVA_HOME="JAVA_HOME_REPLACE" +export HADOOP_HOME="HADOOP_HOME_REPLACE" + +#export HADOOP_ROOT_LOGGER=INFO,DRFA +export HADOOP_SECURITY_LOGGER=INFO,DRFAS +export HDFS_AUDIT_LOGGER=INFO,DRFAAUDIT + +export JSVC_HOME=$HADOOP_HOME/bin +export HADOOP_CLASSPATH=$HADOOP_HOME/lib/native:$HADOOP_CLASSPATH +# default heapsize for hadoop daemon +export HADOOP_HEAPSIZE=5120 +export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" +export HADOOP_LOG_DIR="YARN_LOG_DIR_REPLACE" +export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_LOG_DIR +export HADOOP_PID_DIR="YARN_PID_DIR_REPLACE" +export HADOOP_SECURE_DN_PID_DIR=$HADOOP_PID_DIR + +export HADOOP_NAMENODE_USER=hdfs +export HADOOP_DATANODE_USER=hdfs +export HADOOP_SECURE_DN_USER=hdfs + +GC_LOG_DATE=`date +%F` +GC_LOG_DIR="GC_LOG_DIR_REPLACE" +export NAMENODE_JVM_OPTS="-Dcom.netease.appname=NameNode -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/namenode.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx71680m -Xms71680m -Xmn20480m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +export DATANODE_JVM_OPTS="-Dcom.netease.appname=DataNode -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/datanode.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx10240m -Xms10240m -Xmn2048m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +###mod + + +# The jsvc implementation to use. Jsvc is required to run secure datanodes. +#export JSVC_HOME=${JSVC_HOME} + +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"} + +# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler. +for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do + if [ "$HADOOP_CLASSPATH" ]; then + export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f + else + export HADOOP_CLASSPATH=$f + fi +done + +# The maximum amount of heap to use, in MB. Default is 1000. +#export HADOOP_HEAPSIZE= +#export HADOOP_NAMENODE_INIT_HEAPSIZE="" + +# Extra Java runtime options. Empty by default. +# export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" + +# Command specific options appended to HADOOP_OPTS when specified +export HADOOP_NAMENODE_OPTS="$NAMENODE_JVM_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS" +export HADOOP_DATANODE_OPTS="$DATANODE_JVM_OPTS -Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS" + +export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +#export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS" +#export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=6002" +#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS" + +# On secure datanodes, user to run the datanode as after dropping privileges +#export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} + +# Where log files are stored. $HADOOP_HOME/logs by default. +#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER + +# Where log files are stored in the secure data environment. +#export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} + +# The directory where pid files are stored. /tmp by default. +# NOTE: this should be set to a directory that can only be written to by +# the user that will run the hadoop daemons. Otherwise there is the +# potential for a symlink attack. +#export HADOOP_PID_DIR=${HADOOP_PID_DIR} +#export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR} + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/hdfs-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/hdfs-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..e5dfce6fac61d79fa32ab70fc2d57c424afe400a --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/hdfs-site.xml @@ -0,0 +1,79 @@ + + + + + + + local.hostname.nn1 + HDFS_NAMENODE_HOSTS1_REPLACE + + + local.hostname.nn2 + HDFS_NAMENODE_HOSTS2_REPLACE + + + + dfs.ha.namenodes.LOCAL_REALM_REPLACE + nn1,nn2 + + + + dfs.namenode.rpc-address.LOCAL_REALM_REPLACE.nn1 + ${local.hostname.nn1}:8020 + + + dfs.namenode.rpc-address.LOCAL_REALM_REPLACE.nn2 + ${local.hostname.nn2}:8020 + + + + dfs.nameservices + LOCAL_REALM_REPLACE + + + + dfs.client.failover.proxy.provider.LOCAL_REALM_REPLACE + org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider + + + + + fs.permissions.umask-mode + 027 + + + + dfs.blocksize + 268435456 + + + + dfs.client.read.shortcircuit + true + + + + dfs.client.read.shortcircuit.streams.cache.size + 1024 + + + + dfs.client.read.shortcircuit.streams.cache.size.expiry.ms + 300000 + + + + dfs.domain.socket.path + /var/lib/hadoop-hdfs/dn_socket + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/mapred-env.sh b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/mapred-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..84bee0b3d3fd300bce995d53df442dce7c2cb8a1 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/mapred-env.sh @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## +## THIS FILE ACTS AS AN OVERRIDE FOR hadoop-env.sh FOR ALL +## WORK DONE BY THE mapred AND RELATED COMMANDS. +## +## Precedence rules: +## +## mapred-env.sh > hadoop-env.sh > hard-coded defaults +## +## MAPRED_xyz > HADOOP_xyz > hard-coded defaults +## + +### +# Job History Server specific parameters +### + +# Specify the max heapsize for the JobHistoryServer. If no units are +# given, it will be assumed to be in MB. +# This value will be overridden by an Xmx setting specified in HADOOP_OPTS, +# and/or MAPRED_HISTORYSERVER_OPTS. +# Default is the same as HADOOP_HEAPSIZE_MAX. +#export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=5120 + +# Specify the JVM options to be used when starting the HistoryServer. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +GC_LOG_DATE=`date +%F` +GC_LOG_DIR="GC_LOG_DIR_REPLACE" +export MAPRED_HISTORYSERVER_OPTS="-server -Xmx5g -Xms5g -Xmn2g -XX:SurvivorRatio=4 -Xss256k -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -Xloggc:${GC_LOG_DIR}/rm.gc.log-${GC_LOG_DATE}" + +# Specify the log4j settings for the JobHistoryServer +# Java property: hadoop.root.logger +#export HADOOP_JHS_LOGGER=INFO,RFA diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/mapred-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/mapred-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..cf03e24460dccc5eb18cac1bea4ab548e980aad3 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/mapred-site.xml @@ -0,0 +1,119 @@ + + + + + + + mapreduce.framework.name + yarn + + + + yarn.app.mapreduce.am.staging-dir + YARN_APP_MAPREDUCE_AM_STAGING_DIR_REPLACE + + + + + mapreduce.jobhistory.address + YARN_JOB_HISTORY_HOST_REPLACE:10020 + + + mapreduce.jobhistory.webapp.address + YARN_JOB_HISTORY_HOST_REPLACE:19888 + + + mapreduce.jobhistory.keytab + MAPRED_KEYTAB_LOCATION_REPLACE + + + mapreduce.jobhistory.principal + mapred/_HOST@${local.realm} + + + + + yarn.app.mapreduce.am.resource.mb + 1536 + + + mapreduce.map.memory.mb + 1024 + + + mapreduce.reduce.memory.mb + 1536 + + + mapreduce.map.java.opts + -Xmx820M + + + + mapreduce.reduce.java.opts + -Xmx1228M + + + yarn.app.mapreduce.am.command-opts + -Xmx1228M + + + + + + io.sort.factor + 100 + + + io.sort.mb + 200 + + + + + mapreduce.map.speculative + false + + + mapreduce.reduce.speculative + false + + + + + mapreduce.map.output.compress + true + + + mapreduce.map.output.compress.codec + com.hadoop.compression.lzo.LzoCodec + + + + yarn.app.mapreduce.am.env + HADOOP_MAPRED_HOME=$HADOOP_HOME + + + mapreduce.map.env + HADOOP_MAPRED_HOME=$HADOOP_HOME + + + mapreduce.reduce.env + HADOOP_MAPRED_HOME=$HADOOP_HOME + + + + mapreduce.job.hdfs-servers + FS_DEFAULTFS_REPLACE + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/resource-types.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/resource-types.xml new file mode 100755 index 0000000000000000000000000000000000000000..54ea7dcf6d678fd47e45ab86ab5a4a00c2c65eb3 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/resource-types.xml @@ -0,0 +1,6 @@ + + + yarn.resource-types + yarn.io/gpu + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/yarn-env.sh b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/yarn-env.sh new file mode 100755 index 0000000000000000000000000000000000000000..41c27e15110af913dc2cba9354a335731d5e3a07 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/yarn-env.sh @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +###mod +export YARN_ROOT_LOGGER=INFO,DRFA +#export YARN_ROOT_LOGGER=DEBUG,DRFA +export YARN_LOG_DIR=$HADOOP_LOG_DIR +export YARN_PID_DIR=$HADOOP_PID_DIR + +GC_LOG_DATE=`date +%F` +GC_LOG_DIR="GC_LOG_DIR_REPLACE" +#export RESOURCEMANAGER_JVM_OPTS="-Dcom.netease.appname=ResourceManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/resourcemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx20480m -Xms20480m -Xmn5120m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +export RESOURCEMANAGER_JVM_OPTS="-Dcom.netease.appname=ResourceManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/resourcemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=6001" +export NODEMANAGER_JVM_OPTS="-Dcom.netease.appname=NodeManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/nodemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -XX:GCLogFileSize=512M -server -Xmx5120m -Xms5120m -Xmn1024m -XX:SurvivorRatio=4 -Xss256k -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+UseParNewGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:-DisableExplicitGC" +#export NODEMANAGER_JVM_OPTS="-Dcom.netease.appname=NodeManager -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/nodemanager.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=6001" +###mod +#export YARN_TIMELINESERVER_OPTS="-Dcom.netease.appname=TimelineServer -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/timelineserver.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=6001" +#export YARN_TIMELINEREADER_OPTS="-Dcom.netease.appname=TimelineReaderServer -Djava.net.preferIPv4Stack=true -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:$GC_LOG_DIR/timelineserver.gc.log.$GC_LOG_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=3 -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=6001" + +# User for YARN daemons +export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} + +# resolve links - $0 may be a softlink +export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}" + +# some Java parameters +# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi + +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# For setting YARN specific HEAP sizes please use this +# Parameter and set appropriately +# YARN_HEAPSIZE=1000 + +# check envvars which might override default args +if [ "$YARN_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" +fi + +# Resource Manager specific parameters + +# Specify the max Heapsize for the ResourceManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_RESOURCEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +#export YARN_RESOURCEMANAGER_HEAPSIZE=1000 + +# Specify the JVM options to be used when starting the ResourceManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +export YARN_RESOURCEMANAGER_OPTS="$RESOURCEMANAGER_JVM_OPTS" + +# Node Manager specific parameters + +# Specify the max Heapsize for the NodeManager using a numerical value +# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set +# the value to 1000. +# This value will be overridden by an Xmx setting specified in either YARN_OPTS +# and/or YARN_NODEMANAGER_OPTS. +# If not specified, the default value will be picked from either YARN_HEAPMAX +# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. +#export YARN_NODEMANAGER_HEAPSIZE=1000 + +# Specify the JVM options to be used when starting the NodeManager. +# These options will be appended to the options specified as YARN_OPTS +# and therefore may override any similar flags set in YARN_OPTS +export YARN_NODEMANAGER_OPTS="$NODEMANAGER_JVM_OPTS" + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + + +# default log directory & file +if [ "$YARN_LOG_DIR" = "" ]; then + YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" +fi +if [ "$YARN_LOGFILE" = "" ]; then + YARN_LOGFILE='yarn.log' +fi + +# default policy file for service-level authorization +if [ "$YARN_POLICYFILE" = "" ]; then + YARN_POLICYFILE="hadoop-policy.xml" +fi + +# restore ordinary behaviour +unset IFS + + +YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" +YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" +YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" +YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" +YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" +fi +YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" diff --git a/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/yarn-site.xml b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/yarn-site.xml new file mode 100755 index 0000000000000000000000000000000000000000..fd5c84075b13ee2251ed6363675a54931463e101 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/etc_secure/hadoop/yarn-site.xml @@ -0,0 +1,652 @@ + + + + + + + local.hostname.rm1 + YARN_RESOURCE_MANAGER_HOSTS1_REPLACE + + + local.hostname.rm2 + YARN_RESOURCE_MANAGER_HOSTS2_REPLACE + + + local.cluster-id + LOCAL_CLUSTER_ID_REPLACE + + + + + yarn.resourcemanager.zk-address + YARN_ZK_ADDRESS_REPLACE + + + yarn.resourcemanager.recovery.enabled + true + + + yarn.resourcemanager.store.class + org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore + + + yarn.resourcemanager.zk-timeout-ms + 90000 + + + yarn.resourcemanager.am.max-attempts + 3 + + + yarn.resourcemanager.state-store.max-completed-applications + 200 + + + + + yarn.resourcemanager.ha.enabled + true + + + yarn.resourcemanager.cluster-id + ${local.cluster-id} + + + yarn.resourcemanager.ha.rm-ids + rm1,rm2 + + + + yarn.resourcemanager.hostname.rm1 + ${local.hostname.rm1} + + + + yarn.resourcemanager.hostname.rm2 + ${local.hostname.rm2} + + + + yarn.resourcemanager.webapp.address.rm1 + ${local.hostname.rm1}:8088 + + + + yarn.resourcemanager.webapp.address.rm2 + ${local.hostname.rm2}:8088 + + + + + yarn.nodemanager.aux-services + mapreduce_shuffle,spark_shuffle,timeline_collector + + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + yarn.nodemanager.aux-services.spark_shuffle.class + org.apache.spark.network.yarn.YarnShuffleService + + + spark.yarn.shuffle.stopOnFailure + true + + + yarn.nodemanager.aux-services.timeline_collector.class + org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService + + + + + yarn.nodemanager.local-dirs + YARN_NODEMANAGER_LOCAL_DIRS_REPLACE + + + yarn.nodemanager.log-dirs + YARN_NODEMANAGER_LOG_DIRS_REPLACE + + + + + Where to aggregate logs in hdfs + yarn.nodemanager.remote-app-log-dir + YARN_AGGREGATED_LOG_DIR_REPLACE + + + yarn.log-aggregation-enable + true + + + yarn.log-aggregation.retain-seconds + 604800 + + + yarn.log-aggregation.retain-check-interval-seconds + 86400 + + + yarn.nodemanager.log-aggregation.compression-type + gz + + + + + yarn.resourcemanager.keytab + YARN_KEYTAB_LOCATION_REPLACE + + + yarn.resourcemanager.principal + yarn/_HOST@${local.realm} + + + + + yarn.nodemanager.keytab + YARN_KEYTAB_LOCATION_REPLACE + + + yarn.nodemanager.principal + yarn/_HOST@${local.realm} + + + + + yarn.nodemanager.container-executor.class + org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor + + + yarn.nodemanager.linux-container-executor.path + /etc/yarn/sbin/Linux-amd64-64/container-executor + + + yarn.nodemanager.linux-container-executor.group + yarn + + + + + Minimum request grant-able by the RM scheduler + yarn.scheduler.minimum-allocation-mb + 512 + + + Maximum request grant-able by the RM scheduler + yarn.scheduler.maximum-allocation-mb + 32768 + + + yarn.scheduler.minimum-allocation-vcores + 1 + + + yarn.scheduler.maximum-allocation-vcores + 30 + + + yarn.resourcemanager.scheduler.class + + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + + yarn.scheduler.increment-allocation-mb + 512 + + + yarn.scheduler.fair.allow-undeclared-pools + false + + + + yarn.client.nodemanager-connect.max-wait-ms + 60000 + + + + + yarn.resourcemanager.client.thread-count + 64 + + + yarn.resourcemanager.resource-tracker.client.thread-count + 64 + + + yarn.resourcemanager.scheduler.client.thread-count + 64 + + + yarn.nodemanager.container-manager.thread-count + 32 + + + yarn.nodemanager.container-metrics.enable + true + Todo hbase metrics cleanup mechanism + + + + + The minimum space that must be available on a disk for + it to be used. This applies to yarn.nodemanager.local-dirs and + yarn.nodemanager.log-dirs. + yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb + 1000 + + + Defines how often NMs wake up to upload log files. + The default value is -1. By default, the logs will be uploaded when + the application is finished. By setting this configure, logs can be uploaded + periodically when the application is running. The minimum rolling-interval-seconds + can be set is 3600. + + yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds + 3600 + + + Enable the node manager to recover after starting + yarn.nodemanager.recovery.enabled + true + + + The local filesystem directory in which the node manager will + store state when recovery is enabled. + yarn.nodemanager.recovery.dir + YARN_NODEMANAGER_RECOVERY_DIR_REPLACE + + + + + yarn.nodemanager.resource.memory-mb + 204800 + + + yarn.nodemanager.resource.cpu-vcores + 40 + + + + yarn.admin.acl + hadoop,yarn + + + yarn.acl.enable + true + + + + + yarn.resourcemanager.nodes.exclude-path + YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH_REPLACE + + + If true, ResourceManager will have proxy-user privileges. + Use case: In a secure cluster, YARN requires the user hdfs delegation-tokens to + do localization and log-aggregation on behalf of the user. If this is set to true, + ResourceManager is able to request new hdfs delegation tokens on behalf of + the user. This is needed by long-running-service, because the hdfs tokens + will eventually expire and YARN requires new valid tokens to do localization + and log-aggregation. Note that to enable this use case, the corresponding + HDFS NameNode has to configure ResourceManager as the proxy-user so that + ResourceManager can itself ask for new tokens on behalf of the user when + tokens are past their max-life-time. + yarn.resourcemanager.proxy-user-privileges.enabled + true + + + + + yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled + false + + + + hadoop.http.filter.initializers + org.apache.hadoop.security.AuthenticationFilterInitializer + + + hadoop.http.authentication.type + + kerberos + + + + yarn.nodemanager.container.use.serialgc.enable + true + + + + + Indicate to clients whether Timeline service is enabled or not. + If enabled, the TimelineClient library used by end-users will post entities + and events to the Timeline server. + yarn.timeline-service.enabled + true + + + + The setting that controls whether yarn system metrics is + published on the timeline server or not by RM. + yarn.resourcemanager.system-metrics-publisher.enabled + true + + + + Indicate to clients whether to query generic application + data from timeline history-service or not. If not enabled then application + data is queried only from Resource Manager. + yarn.timeline-service.generic-application-history.enabled + true + + + + yarn.timeline-service.http-authentication.type + + kerberos + + + + yarn.timeline-service.principal + yarn/_HOST@${local.realm} + + + + yarn.timeline-service.keytab + YARN_KEYTAB_LOCATION_REPLACE + + + + yarn.timeline-service.http-authentication.kerberos.principal + HTTP/_HOST@${local.realm} + + + + yarn.timeline-service.http-authentication.kerberos.keytab + HTTP_KEYTAB_LOCATION_REPLACE + + + + yarn.timeline-service.hostname + YARN_TIMELINE_HOST_REPLACE + + + + yarn.timeline-service.hbase.coprocessor.jar.hdfs.location + /hbase/coprocessor + + + + URL for log aggregation server web service + yarn.log.server.web-service.url + http://YARN_TIMELINE_HOST_REPLACE:8189/ws/v2/applicationlog + + + + + yarn.timeline-service.leveldb-timeline-store.path + YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_REPLACE + + + + yarn.timeline-service.leveldb-state-store.path + YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_REPLACE + + + + yarn.timeline-service.leveldb-timeline-store.read-cache-size + 104857600 + + + + yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size + 10000 + + + + yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size + 10000 + + + + yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms + 300000 + + + + yarn.timeline-service.recovery.enabled + true + + + + yarn.timeline-service.state-store-class + org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore + + + + yarn.timeline-service.store-class + org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore + + + + yarn.timeline-service.ttl-enable + true + + + + yarn.timeline-service.ttl-ms + 2678400000 + + + + + yarn.timeline-service.version + 2.0f + + + + The setting that controls whether yarn system metrics is + published on the Timeline service or not by RM And NM. + yarn.system-metrics-publisher.enabled + true + + + + The setting that controls whether yarn container events are + published to the timeline service or not by RM. This configuration setting + is for ATS V2. + + yarn.rm.system-metrics-publisher.emit-container-events + true + + + + Optional URL to an hbase-site.xml configuration file to be + used to connect to the timeline-service hbase cluster. If empty or not + specified, then the HBase configuration will be loaded from the classpath. + When specified the values in the specified configuration file will override + those from the ones that are present on the classpath. + + yarn.timeline-service.hbase.configuration.file + YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE_REPLACE + + + + + yarn.timeline-service.versions + 1.5f,2.0f + + + + yarn.timeline-service.entity-group-fs-store.active-dir + YARN_TIMELINE_FS_STORE_DIR_REPLACE/active + + + + yarn.timeline-service.entity-group-fs-store.done-dir + YARN_TIMELINE_FS_STORE_DIR_REPLACE/done + + + + + yarn.timeline-service.reader.webapp.address + YARN_TIMELINE_HOST_REPLACE:8189 + + + + yarn.timeline-service.entity-group-fs-store.summary-store + org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore + + + + Enable services rest api on ResourceManager. + yarn.webapp.api-service.enable + true + + + + The domain name for Hadoop cluster associated records. As short as possible. + hadoop.registry.dns.domain-name + ml + + + + The DNS functionality is enabled for the cluster. Default is false. + hadoop.registry.dns.enabled + true + + + Address associated with the network interface to which the DNS listener should bind. + hadoop.registry.dns.bind-address + YARN_REGISTRY_DNS_HOST_REPLACE + + + The port number for the DNS listener. The default port is 5353. + If the standard privileged port 53 is used, make sure start the DNS with jsvc support. + hadoop.registry.dns.bind-port + + YARN_REGISTRY_DNS_HOST_PORT_REPLACE + + + + + The Kerberos keytab file to be used for spnego filter for the NM web interface. + + yarn.nodemanager.webapp.spnego-keytab-file + HTTP_KEYTAB_LOCATION_REPLACE + + + + + The Kerberos principal to be used for spnego filter for the NM web interface. + + yarn.nodemanager.webapp.spnego-principal + HTTP/_HOST@${local.realm} + + + + + The Kerberos keytab file to be used for spnego filter for the RM web interface. + + yarn.resourcemanager.webapp.spnego-keytab-file + HTTP_KEYTAB_LOCATION_REPLACE + + + + + The Kerberos principal to be used for spnego filter for the RM web interface. + + yarn.resourcemanager.webapp.spnego-principal + HTTP/_HOST@${local.realm} + + + + + yarn.nodemanager.linux-container-executor.resources-handler.class + org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler + + + yarn.nodemanager.linux-container-executor.cgroups.mount + false + + + yarn.nodemanager.resource.percentage-physical-cpu-limit + 80 + + + Whether virtual memory limits will be enforced for containers. + yarn.nodemanager.vmem-check-enabled + false + + + + Whether YARN CGroups memory tracking is enabled. + yarn.nodemanager.resource.memory.enabled + true + + + + + yarn.nodemanager.runtime.linux.docker.default-container-network + YARN_DOCKER_CONTAINER_DEFAULT_NETWORK_REPLACE + + + yarn.nodemanager.runtime.linux.allowed-runtimes + default,docker + + + yarn.nodemanager.runtime.linux.docker.allowed-container-networks + YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS_REPLACE + + + yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed + false + + + yarn.nodemanager.runtime.linux.docker.privileged-containers.acl + + + + yarn.nodemanager.runtime.linux.docker.capabilities + CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE + + + + + yarn.webapp.ui2.enable + true + + + + + + yarn.service.base.path + /tmp/.LOCAL_CLUSTER_ID_REPLACE/ + + + + yarn.nodemanager.address + 0.0.0.0:45454 + + + diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-historyserver.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-historyserver.sh new file mode 100644 index 0000000000000000000000000000000000000000..ed84feaa8f67835640a71e7ee2d79291658faf81 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-historyserver.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "mapred" ];then + echo "[ERROR]:jobhistory must be started with the [mapred] user!" +fi + +YARN_LOGFILE=mr-historyserver.log ${common_bin}/mr-jobhistory-daemon.sh start historyserver diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-mr-jobhistory.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-mr-jobhistory.sh new file mode 100755 index 0000000000000000000000000000000000000000..5d54144472a7d10d19ac09718a94577f5c7b61b7 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-mr-jobhistory.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` +YARN_LOGFILE=mr-jobhistory.log ${common_bin}/mr-jobhistory-daemon.sh start historyserver diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-nodemanager.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-nodemanager.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c5cac2824e87ff76b764773f5a3ebea3aa60248 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-nodemanager.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:nodemanager must be started with the [yarn] user!" +fi + +YARN_LOGFILE=nodemanager.log ${common_bin}/yarn-daemon.sh start nodemanager diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-registrydns.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-registrydns.sh new file mode 100644 index 0000000000000000000000000000000000000000..8d2aa7c8895dbb41758401ee6f5e1f51a8c84ba5 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-registrydns.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "root" ];then + echo "[ERROR]:Registrydns must be started with the [root] user!" +fi + +JSVC_HOME=${common_bin}/../bin YARN_LOGFILE=registrydns.log ${common_bin}/yarn-daemon.sh start registrydns diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-resourcemanager.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-resourcemanager.sh new file mode 100644 index 0000000000000000000000000000000000000000..217b2bf0e40624cc2401a671d3736f9a851c45f8 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-resourcemanager.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:resourcemanager must be started with the [yarn] user!" +fi + +YARN_LOGFILE=resourcemanager.log ${common_bin}/yarn-daemon.sh start resourcemanager diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-timelinereader.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-timelinereader.sh new file mode 100644 index 0000000000000000000000000000000000000000..33e1586089780a69f48e96ef206ad352f1837a41 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-timelinereader.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:timelinereader must be started with the [yarn] user!" +fi + +YARN_LOGFILE=timelinereader.log ${common_bin}/yarn-daemon.sh start timelinereader diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-timelineserver.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-timelineserver.sh new file mode 100644 index 0000000000000000000000000000000000000000..fbe45be24f4cc41c88cac71cb69991a0a27c9276 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/start-timelineserver.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:timelineserver must be started with the [yarn] user!" +fi + +HADOOP_SHELL_SCRIPT_DEBUG=true YARN_LOGFILE=timelineserver.log ${common_bin}/yarn-daemon.sh start timelineserver diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-historyserver.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-historyserver.sh new file mode 100755 index 0000000000000000000000000000000000000000..133ff5d2ea56c9ba127b727bb95a4faa6638e012 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-historyserver.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "mapred" ];then + echo "[ERROR]:jobhistory must be started with the [mapred] user!" +fi + +YARN_LOGFILE=mr-historyserver.log ${common_bin}/mr-jobhistory-daemon.sh stop historyserver diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-mr-jobhistory.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-mr-jobhistory.sh new file mode 100755 index 0000000000000000000000000000000000000000..98b9ae0435dd5feca29c4b25e3bb5d0bfeb755d7 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-mr-jobhistory.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` +YARN_LOGFILE=mr-jobhistory.log ${common_bin}/mr-jobhistory-daemon.sh stop historyserver diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-nodemanager.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-nodemanager.sh new file mode 100755 index 0000000000000000000000000000000000000000..64261ccf2b0c8273fe67629e08079ab6d3182e27 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-nodemanager.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:nodemanager must be started with the [yarn] user!" +fi + +YARN_LOGFILE=nodemanager.log ${common_bin}/yarn-daemon.sh stop nodemanager diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-registrydns.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-registrydns.sh new file mode 100755 index 0000000000000000000000000000000000000000..a6f81c1eb64017e0e553c50137a4b383da5ccdf7 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-registrydns.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "root" ];then + echo "[ERROR]:Registrydns must be started with the [root] user!" +fi + +JSVC_HOME=${common_bin}/../bin YARN_LOGFILE=registrydns.log ${common_bin}/yarn-daemon.sh stop registrydns diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-resourcemanager.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-resourcemanager.sh new file mode 100755 index 0000000000000000000000000000000000000000..9f1b8cfe617c8add03fa98e5cd1a0cd139f596c9 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-resourcemanager.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:resourcemanager must be started with the [yarn] user!" +fi + +YARN_LOGFILE=resourcemanager.log ${common_bin}/yarn-daemon.sh stop resourcemanager diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-timelinereader.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-timelinereader.sh new file mode 100755 index 0000000000000000000000000000000000000000..17eb8d9709ae9d4a4bd32cf3f61d1ccabc82f060 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-timelinereader.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:timelinereader must be started with the [yarn] user!" +fi + +YARN_LOGFILE=timelinereader.log ${common_bin}/yarn-daemon.sh stop timelinereader diff --git a/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-timelineserver.sh b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-timelineserver.sh new file mode 100755 index 0000000000000000000000000000000000000000..071ee77ad13eea62bcb5e2693029cd83533c4864 --- /dev/null +++ b/dev-support/submarine-installer/package/hadoop/yarn/sbin/stop-timelineserver.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd `dirname $0` +common_bin=`pwd` + +if [ `whoami` != "yarn" ];then + echo "[ERROR]:timelineserver must be started with the [yarn] user!" +fi + +HADOOP_SHELL_SCRIPT_DEBUG=true YARN_LOGFILE=timelineserver.log ${common_bin}/yarn-daemon.sh stop timelineserver diff --git a/dev-support/submarine-installer/package/submarine/submarine.sh b/dev-support/submarine-installer/package/submarine/submarine.sh new file mode 100644 index 0000000000000000000000000000000000000000..b5c4b039760d97f27671b0cdcec54be1c467af4e --- /dev/null +++ b/dev-support/submarine-installer/package/submarine/submarine.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Solution: Grant user yarn the access to /sys/fs/cgroup/cpu,cpuacct, +# which is the subfolder of cgroup mount destination. +chown :yarn -R /sys/fs/cgroup/cpu,cpuacct +chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct + +# Grant user yarn the access to /sys/fs/cgroup/memory, +# which is the subfolder of cgroup mount destination. +chown :yarn -R /sys/fs/cgroup/memory +chmod g+rwx -R /sys/fs/cgroup/memory + + +# If GPUs are used，the access to cgroup devices folder is needed as well +chown :yarn -R /sys/fs/cgroup/devices +chmod g+rwx -R /sys/fs/cgroup/devices diff --git a/dev-support/submarine-installer/scripts/calico.sh b/dev-support/submarine-installer/scripts/calico.sh new file mode 100644 index 0000000000000000000000000000000000000000..b3eb80be1b5ee6c9da09ff8ab1feefdc89d68fcc --- /dev/null +++ b/dev-support/submarine-installer/scripts/calico.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download calico bin +## @audience public +## @stability stable +function download_calico_bin() +{ + # submarine http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_CALICOCTL_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calicoctl + MY_CALICO_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico + MY_CALICO_IPAM_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico-ipam + else + MY_CALICOCTL_DOWNLOAD_URL=${CALICOCTL_DOWNLOAD_URL} + MY_CALICO_DOWNLOAD_URL=${CALICO_DOWNLOAD_URL} + MY_CALICO_IPAM_DOWNLOAD_URL=${CALICO_IPAM_DOWNLOAD_URL} + fi + + mkdir -p "${DOWNLOAD_DIR}/calico" + + if [[ -f ${DOWNLOAD_DIR}/calico/calico ]]; then + echo "${DOWNLOAD_DIR}/calico/calico already exists." + else + echo "download ${MY_CALICO_DOWNLOAD_URL} ..." + wget -P "${DOWNLOAD_DIR}/calico" "${MY_CALICO_DOWNLOAD_URL}" + fi + + if [[ -f ${DOWNLOAD_DIR}/calico/calicoctl ]]; then + echo "${DOWNLOAD_DIR}/calico already exists." + else + echo "download ${MY_CALICOCTL_DOWNLOAD_URL} ..." + wget -P "${DOWNLOAD_DIR}/calico" "${MY_CALICOCTL_DOWNLOAD_URL}" + fi + + if [[ -f ${DOWNLOAD_DIR}/calico/calico-ipam ]]; then + echo "${DOWNLOAD_DIR}/calico/calico-ipam already exists." + else + echo "download ${MY_CALICO_IPAM_DOWNLOAD_URL} ..." + wget -P "${DOWNLOAD_DIR}/calico" "${MY_CALICO_IPAM_DOWNLOAD_URL}" + fi +} + +## @description install calico bin +## @audience public +## @stability stable +function install_calico_bin() +{ + download_calico_bin + + sudo cp -f "${DOWNLOAD_DIR}/calico/calico" /usr/bin/calico + sudo cp -f "${DOWNLOAD_DIR}/calico/calicoctl" /usr/bin/calicoctl + sudo cp -f "${DOWNLOAD_DIR}/calico/calico-ipam" /usr/bin/calico-ipam + + sudo chmod +x /usr/bin/calico + sudo chmod +x /usr/bin/calicoctl + sudo chmod +x /usr/bin/calico-ipam +} + +## @description install calico config +## @audience public +## @stability stable +function install_calico_config() +{ + sudo mkdir -p /etc/calico + + cp -rf "${PACKAGE_DIR}/calico" "${INSTALL_TEMP_DIR}/" + + # 1. replace etcdEndpoints + # etcdEndpoints: https://10.196.69.173:2379,https://10.196.69.174:2379,https://10.196.69.175:2379 + etcdEndpoints='' + index=0 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in "${ETCD_HOSTS[@]}" + do + index=$((index+1)) + etcdEndpoints="${etcdEndpoints}http:\\/\\/${item}:2379" + if [[ ${index} -lt ${etcdHostsSize} ]]; then + etcdEndpoints=${etcdEndpoints}"," + fi + done + # echo "etcdEndpoints=${etcdEndpoints}" + sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" "$INSTALL_TEMP_DIR/calico/calicoctl.cfg" + + if [[ ! -d /etc/calico ]]; then + sudo mkdir /etc/calico + else + sudo rm -rf /etc/calico/* + fi + + sudo cp -f "$INSTALL_TEMP_DIR/calico/calicoctl.cfg" /etc/calico/calicoctl.cfg + + sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" "$INSTALL_TEMP_DIR/calico/calico-node.service" + sed -i "s/CALICO_IPV4POOL_CIDR_REPLACE/${CALICO_IPV4POOL_CIDR}/g" "$INSTALL_TEMP_DIR/calico/calico-node.service" + sudo cp "$INSTALL_TEMP_DIR/calico/calico-node.service" /etc/systemd/system/ + + sudo systemctl daemon-reload + sudo systemctl enable calico-node.service +} + +## @description modify kernel network config +## @audience public +## @stability stable +function kernel_network_config() +{ + if [ "$(grep -c "net.ipv4.conf.all.rp_filter=1" /etc/sysctl.conf)" -eq '0' ]; then + sudo echo "net.ipv4.conf.all.rp_filter=1" >>/etc/sysctl.conf + fi + + if [ "$(grep -c "net.ipv4.ip_forward=1" /etc/sysctl.conf)" -eq '0' ]; then + sudo echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf + fi + + sudo sysctl -p +} + +## @description check if the calico-network exist +## @audience public +## @stability stable +function calico_network_exist() +{ + local dockerNetworkInfo + dockerNetworkInfo=$(docker network ls --filter NAME="${CALICO_NETWORK_NAME}") + echo "${dockerNetworkInfo}" | grep "${CALICO_NETWORK_NAME}" +} + +## @description verification calico +## @audience public +## @stability stable +function verification_calico() +{ + echo " ===== Check if the network between 2 containers can be connected =====" + local calicoNetworkExist + calicoNetworkExist=$(calico_network_exist) + if [[ "$calicoNetworkExist" = "" ]]; then + echo "Create a calico network" + docker network create --driver calico --ipam-driver calico-ipam "${CALICO_NETWORK_NAME}" + else + echo "calico network ${CALICO_NETWORK_NAME} already exists." + fi + + local verifyA="verify-calico-network-A" + local verifyAInfo + verifyAInfo=$(containers_exist ${verifyA}) + if [[ -n "$verifyAInfo" ]]; then + echo "Delete existing container ${verifyA}." + docker stop ${verifyA} + docker rm ${verifyA} + fi + echo "Create containers verify-calico-network-A" + docker run --net "${CALICO_NETWORK_NAME}" --name ${verifyA} -tid busybox + + local verifyB="verify-calico-network-B" + local verifyBInfo + verifyBInfo=$(containers_exist ${verifyB}) + if [[ -n "$verifyBInfo" ]]; then + echo "Delete existing container ${verifyB}." + docker stop ${verifyB} + docker rm ${verifyB} + fi + echo "Create containers verify-calico-network-B" + docker run --net "${CALICO_NETWORK_NAME}" --name ${verifyB} -tid busybox + + echo -e "\\033[33m${verifyA} ping ${verifyB}\\033[0m" + docker exec ${verifyA} ping ${verifyB} -c 5 +} + +## @description install calico +## @audience public +## @stability stable +function install_calico() +{ + kernel_network_config + install_calico_bin + install_calico_config + start_calico + verification_calico +} + +## @description uninstall calico +## @audience public +## @stability stable +function uninstall_calico() +{ + echo "stop calico-node.service" + sudo systemctl stop calico-node.service + + echo "rm /usr/bin/calico ..." + sudo rm /usr/bin/calicoctl + sudo rm /usr/bin/calico + sudo rm /usr/bin/calico-ipam + + sudo rm -rf /etc/calico/ + sudo rm /etc/systemd/system/calico-node.service + systemctl daemon-reload +} + +## @description start calico +## @audience public +## @stability stable +function start_calico() +{ + sudo systemctl restart calico-node.service + sudo systemctl status calico-node.service +} + +## @description stop calico +## @audience public +## @stability stable +function stop_calico() +{ + sudo systemctl stop calico-node.service + sudo systemctl status calico-node.service +} diff --git a/dev-support/submarine-installer/scripts/combine-docker-daemons.py b/dev-support/submarine-installer/scripts/combine-docker-daemons.py new file mode 100644 index 0000000000000000000000000000000000000000..5d5ff58d4ec0d9a9774a52b99cb449eb64505265 --- /dev/null +++ b/dev-support/submarine-installer/scripts/combine-docker-daemons.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import sys + +def combineJsons(jsonFile1, jsonFile2, outputFile): + dict1 = json.load(open(jsonFile1)) + dict2 = json.load(open(jsonFile2)) + dict3 = dict(dict1.items() + dict2.items()) + + with open(outputFile, 'w') as output: + json.dump(dict3, output, indent=2, sort_keys=True) + + return True + +if __name__ == '__main__': + if (len(sys.argv) < 4): + raise Exception,u"3 arguments needed" + + print(combineJsons(sys.argv[1], sys.argv[2], sys.argv[3])) diff --git a/dev-support/submarine-installer/scripts/docker.sh b/dev-support/submarine-installer/scripts/docker.sh new file mode 100644 index 0000000000000000000000000000000000000000..c282a2a8d38e931f4ba01da72799406cd7553008 --- /dev/null +++ b/dev-support/submarine-installer/scripts/docker.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download docker rmp +## @audience public +## @stability stable +function download_docker_rpm() +{ + # download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_DOCKER_ENGINE_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_RPM}" + else + # Trim the last slash of DOCKER_REPO + DOCKER_REPO_TRIMED="$(echo -e "${DOCKER_REPO}" | sed -e 's/\/*$//')" + MY_DOCKER_ENGINE_RPM=${DOCKER_REPO_TRIMED}/${DOCKER_ENGINE_RPM} + fi + + if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} ]]; then + echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} already exists." + else + echo "download ${MY_DOCKER_ENGINE_RPM} ..." + wget -P "${DOWNLOAD_DIR}/docker/" "${MY_DOCKER_ENGINE_RPM}" + if [[ $? -ne 0 ]]; then + echo -e "\\033[32mshell:> Failed to download ${DOCKER_ENGINE_RPM} of docker + from ${MY_DOCKER_ENGINE_RPM} \\033[0m" + fi + fi +} + +## @description install docker bin +## @audience public +## @stability stable +function install_docker_bin() +{ + download_docker_rpm + + sudo yum -y localinstall "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM}" +} + +## @description uninstall docker bin +## @audience public +## @stability stable +function uninstall_docker_bin() +{ + sudo yum -y remove "${DOCKER_VERSION}" +} + +## @description install docker config +## @audience public +## @stability stable +function install_docker_config() +{ + rm -rf "${INSTALL_TEMP_DIR}/docker" + cp -rf "${PACKAGE_DIR}/docker" "${INSTALL_TEMP_DIR}/" + + # replace cluster-store + # "cluster-store":"etcd://10.196.69.173:2379,10.196.69.174:2379,10.196.69.175:2379" + # char '/' need to escape '\/' + clusterStore="etcd:\\/\\/" + index=1 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in "${ETCD_HOSTS[@]}" + do + clusterStore="${clusterStore}${item}:2379" + if [[ ${index} -lt ${etcdHostsSize} ]]; then + clusterStore=${clusterStore}"," + fi + index=$((index+1)) + done + # echo "clusterStore=${clusterStore}" + sed -i "s/CLUSTER_STORE_REPLACE/${clusterStore}/g" "$INSTALL_TEMP_DIR/docker/daemon.json" + + sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" "$INSTALL_TEMP_DIR/docker/daemon.json" + sed -i "s/LOCAL_HOST_IP_REPLACE/${LOCAL_HOST_IP}/g" "$INSTALL_TEMP_DIR/docker/daemon.json" + YARN_REGISTRY_DNS_IP=$(getent hosts "${YARN_REGISTRY_DNS_HOST}" | awk '{ print $1 }') + sed -i "s/YARN_REGISTRY_DNS_HOST_REPLACE/${YARN_REGISTRY_DNS_IP}/g" "$INSTALL_TEMP_DIR/docker/daemon.json" + + hosts=${LOCAL_DNS_HOST//,/ } + hosts_length=0 + for element in $hosts + do + hosts_length=$((${hosts_length} + 1)) + if [ ${hosts_length} != 1 ]; then + NEW_LOCAL_DNS_HOST="${NEW_LOCAL_DNS_HOST}, " + fi + NEW_LOCAL_DNS_HOST="${NEW_LOCAL_DNS_HOST}\"${element}\"" + done + sed -i "s/LOCAL_DNS_HOST_REPLACE/${NEW_LOCAL_DNS_HOST}/g" "$INSTALL_TEMP_DIR/docker/daemon.json" + + # Delete the ASF license comment in the daemon.json file, otherwise it will cause a json format error. + sed -i '1,16d' "$INSTALL_TEMP_DIR/docker/daemon.json" + + if [ ! -d "/etc/docker" ]; then + sudo mkdir /etc/docker + fi + + sudo cp "$INSTALL_TEMP_DIR/docker/daemon.json" /etc/docker/ + sudo cp "$INSTALL_TEMP_DIR/docker/docker.service" /etc/systemd/system/ + + # Change docker store path + if [[ -n "${DOCKER_STORE_PATH}" ]]; then + mkdir -p "${DOCKER_STORE_PATH}" + cp -r /var/lib/docker/* "${DOCKER_STORE_PATH}" + rm -rf /var/lib/docker + ln -s "${DOCKER_STORE_PATH}" "/var/lib/docker" + fi +} + +## @description install docker +## @audience public +## @stability stable +function install_docker() +{ + install_docker_bin + install_docker_config + + sudo systemctl daemon-reload + sudo systemctl enable docker.service +} + +## @description uninstall docker +## @audience public +## @stability stable +function uninstall_docker() +{ + echo "stop docker service" + sudo systemctl stop docker + + echo "remove docker" + uninstall_docker_bin + + sudo rm /etc/systemd/system/docker.service + + sudo systemctl daemon-reload +} + +## @description start docker +## @audience public +## @stability stable +function start_docker() +{ + sudo systemctl restart docker + sudo systemctl status docker + docker info +} + +## @description stop docker +## @audience public +## @stability stable +function stop_docker() +{ + sudo systemctl stop docker + sudo systemctl status docker +} + +## @description check if the containers exist +## @audience public +## @stability stable +function containers_exist() +{ + local dockerContainersInfo + dockerContainersInfo=$(docker ps -a --filter NAME="$1") + echo "${dockerContainersInfo}" | grep "$1" +} diff --git a/dev-support/submarine-installer/scripts/download-server.sh b/dev-support/submarine-installer/scripts/download-server.sh new file mode 100644 index 0000000000000000000000000000000000000000..244d7a880cb8e50067f7ce55cb327f680c2023c5 --- /dev/null +++ b/dev-support/submarine-installer/scripts/download-server.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description start download server +## @audience public +## @stability stable +function start_download_server() +{ + if [[ -n "$DOWNLOAD_SERVER_IP" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then + echo -e "\\033[31mERROR: Only $DOWNLOAD_SERVER_IP can start the download service.\\033[0m" + return 1 + fi + + echo -e "You can put the install package file in the \\033[34m${DOWNLOAD_DIR}\\033[0m folder first, Or automatic download." + echo -n "Do you want to start download http server?[y|n]" + read -r myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + IS_DOWNLOAD_SERVER=true + download_etcd_bin + download_calico_bin + download_docker_rpm + download_nvidia_driver + download_nvidia_docker_bin + download_yarn_container_executor + + python -m SimpleHTTPServer "${DOWNLOAD_SERVER_PORT}" + fi +} diff --git a/dev-support/submarine-installer/scripts/environment.sh b/dev-support/submarine-installer/scripts/environment.sh new file mode 100644 index 0000000000000000000000000000000000000000..916695c1313d56c0bc01eabd39e86cbbe2c0ad95 --- /dev/null +++ b/dev-support/submarine-installer/scripts/environment.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description check operation System +## @audience public +## @stability stable +function check_operationSystem() +{ + echo -e "The submarine assembly support \\033[32m[centos-release-7-5.1804.el7.centos.x86_64]\\033[0m or higher operating system version." + + case ${OPERATING_SYSTEM} in + centos) + local operationSystemVersion + operationSystemVersion=$(rpm --query centos-release) + echo -e "The current operating system version is \\e[31m[${operationSystemVersion}]\\e[0m" + ;; + *) + echo -e "\\033[31mWARN: The submarine assembly Unsupported [${OPERATING_SYSTEM}] operating system\\033[0m" + ;; + esac +} + +## @description update operation System Kernel +## @audience public +## @stability stable +function update_operationSystemKernel() +{ + echo "If the server is unable to connect to the network, execute the following command yourself: + wget http://vault.centos.org/7.5.1804/os/x86_64/Packages/kernel-headers-3.10.0-862.el7.x86_64.rpm + rpm -ivh kernel-headers-3.10.0-862.el7.x86_64.rpm" + + echo -n "Do you want to kernel upgrades?[y|n]" + read -r myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + echo "Now try to use the yum command for kernel upgrades ..." + yum install "kernel-devel-$(uname -r)" "kernel-headers-$(uname -r)" + + local kernelVersion + kernelVersion=$(uname -r) + echo -e "After the upgrade, the operating system kernel version is \\e[31m${kernelVersion}\\e[0m" + fi +} + +## @description check operation system kernel +## @audience public +## @stability stable +function check_operationSystemKernel() +{ +case ${OPERATING_SYSTEM} in +centos) + local kernelVersion + kernelVersion=$(uname -r) + + echo -e "Submarine support operating system kernel version is \\033[32m 3.10.0-862.el7.x86_64 \\033[0m" + echo -e "Current operating system kernel version is \\e[31m${kernelVersion}\\e[0m" + + update_operationSystemKernel + ;; +*) + echo -e "\\033[31m WARN: The submarine assembly Unsupported operating system [${OPERATING_SYSTEM}] \\033[0m" + ;; +esac +} + +## @description get gcc version +## @audience public +## @stability stable +function get_gcc_version() +{ + local gccVersion + gccVersion=$(gcc --version) + version=${gccVersion%Copyright*} + echo "$version" +} + +## @description install gcc +## @audience public +## @stability stable +function install_gcc() +{ + echo -n "Do you want to install gcc?[y|n]" + read -r myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]]; then + echo "Execute the yum install gcc make g++ command" + yum install gcc make g++ + + local gccVersion + gccVersion=$(gcc --version) + echo -e "After the install, the gcc version is \\e[31m${gccVersion}\\e[0m" + fi +} + +## @description check gcc Version +## @audience public +## @stability stable +function check_gccVersion() +{ + local gccVersionInfo + gccVersionInfo=$(gcc --version) + local gccVersion=${gccVersionInfo%Copyright*} + + if [[ "$gccVersion" = "" ]]; then + echo "The gcc was not installed on the system. Automated installation ..." + install_gcc + else + echo -e "Submarine gcc version need \\033[34mgcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\\033[0m or higher." + echo -e "Current gcc version was \\033[34m${gccVersion}\\033[0m" + fi +} + +## @description check GPU +## @audience public +## @stability stable +function check_GPU() +{ + gpuInfo=$(lspci | grep -i nvidia) + + if [[ "$gpuInfo" = "" ]]; then + echo -e "\\033[31mERROR: The system did not detect the GPU graphics card.\\033[0m" + else + echo -e "\\033[32mINFO: The system detect the GPU graphics card.\\033[0m" + fi +} + +## @description check user group +## @audience public +## @stability stable +function check_userGroup() +{ + echo -e "check hadoop user group ..." + + echo -e "Hadoop runs the required user [hdfs, mapred, yarn] and groups [hdfs, mapred, yarn, hadoop] installed by ambari." + echo -e "If you are not using ambari for hadoop installation, +then you can add the user and group by root by executing the following command: +\\033[34madduser hdfs +adduser mapred +adduser yarn +addgroup hadoop +usermod -aG hdfs,hadoop hdfs +usermod -aG mapred,hadoop mapred +usermod -aG yarn,hadoop yarn +usermod -aG hdfs,hadoop hadoop +groupadd docker +usermod -aG docker yarn +usermod -aG docker hadoop\\033[0m\\n" + + echo -e "check docker user group ..." + # check user group + DOCKER_USER_GROUP='docker' + + if ! grep -E "^${DOCKER_USER_GROUP}" /etc/group >& /dev/null + then + echo -e "user group ${DOCKER_USER_GROUP} does not exist, Please execute the following command:" + echo -e "\\033[34mgroupadd $DOCKER_USER_GROUP\\033[0m" + fi + + # check user + USER_GROUP=(yarn hadoop) + for user in "${USER_GROUP[@]}" + do + if ! grep -E "^${user}" /etc/passwd >& /dev/null + then + echo -e "User ${user} does not exist, Please execute the following command:" + echo -e "\\033[34madduser ${user}\\033[0m" + echo -e "\\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\\033[0m" + fi + + echo -e "Please execute the following command:" + echo -e "\\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\\033[0m" + done +} + +## @description Some preparatory work for nvidia driver installation +## @audience public +## @stability stable +function prepare_nvidia_environment() +{ + echo "prepare nvidia environment ..." + + yum install "kernel-devel-$(uname -r)" "kernel-headers-$(uname -r)" + + yum -y install epel-release + yum -y install dkms + + echo -e "\\033[34m ===== Please manually execute the following command ===== +# 1. Disable nouveau +# Add the content 'rd.driver.blacklist=nouveau nouveau.modeset=0' +# to the 'GRUB_CMDLINE_LINUX' configuration item in the /etc/default/grub file. +root:> vi /etc/default/grub +vi:> GRUB_CMDLINE_LINUX=\"rd.driver.blacklist=nouveau nouveau.modeset=0 ...\" + +# 2. Generate configuration +root:> grub2-mkconfig -o /boot/grub2/grub.cfg + +# 3. Open (new) /etc/modprobe.d/blacklist.conf, add content 'blacklist nouveau' +root:> vi /etc/modprobe.d/blacklist.conf +vi:> blacklist nouveau + +# 4. Update configuration and reboot +root:> mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r)-nouveau.img +root:> dracut /boot/initramfs-$(uname -r).img $(uname -r) +root:> reboot +\\033[0m" +} diff --git a/dev-support/submarine-installer/scripts/etcd.sh b/dev-support/submarine-installer/scripts/etcd.sh new file mode 100644 index 0000000000000000000000000000000000000000..00bb10ccace8450b6889bbddc07ee0ea15dec17b --- /dev/null +++ b/dev-support/submarine-installer/scripts/etcd.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download etcd bin +## @audience public +## @stability stable +function download_etcd_bin() +{ + # my download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_ETCD_DOWNLOAD_URL="${DOWNLOAD_HTTP}/downloads/etcd/${ETCD_TAR_GZ}" + else + MY_ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL} + fi + + if [[ -f "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" ]]; then + echo "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} already exists." + else + echo "download ${MY_ETCD_DOWNLOAD_URL} ..." + wget -P "${DOWNLOAD_DIR}/etcd" "${MY_ETCD_DOWNLOAD_URL}" + fi +} + +## @description install etcd bin +## @audience public +## @stability stable +function install_etcd_bin() +{ + download_etcd_bin + + # install etcd bin + mkdir -p "${INSTALL_TEMP_DIR}" + rm -rf "${INSTALL_TEMP_DIR}/etcd-*-linux-amd6" + tar zxvf "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" -C "${INSTALL_TEMP_DIR}" + + cp -f "${INSTALL_TEMP_DIR}/etcd-${ETCD_VERSION}-linux-amd64/etcd" /usr/bin + cp -f "${INSTALL_TEMP_DIR}/etcd-${ETCD_VERSION}-linux-amd64/etcdctl" /usr/bin + + mkdir -p /var/lib/etcd + chmod -R a+rw /var/lib/etcd +} + +## @description install etcd config +## @audience public +## @stability stable +function install_etcd_config() +{ + # config etcd.service + rm -rf "${INSTALL_TEMP_DIR}/etcd" + cp -rf "${PACKAGE_DIR}/etcd" "${INSTALL_TEMP_DIR}/" + + # 1. Replace name with ETCD_NODE_NAME_REPLACE based on the location of the local IP in $ETCD_HOSTS + indexEtcdList=$(indexByEtcdHosts "${LOCAL_HOST_IP}") + # echo ${indexEtcdList} + etcdNodeName="etcdnode${indexEtcdList}" + # echo ${etcdNodeName} + sed -i "s/ETCD_NODE_NAME_REPLACE/${etcdNodeName}/g" "$INSTALL_TEMP_DIR/etcd/etcd.service" + + # 2. Replace local IP address + sed -i "s/LOCAL_HOST_REPLACE/${LOCAL_HOST_IP}/g" "$INSTALL_TEMP_DIR/etcd/etcd.service" + + # 3. Replace the initial-cluster parameter + # --initial-cluster=etcdnode1=http://10.196.69.173:2380,etcdnode2=http://10.196.69.174:2380,etcdnode3=http://10.196.69.175:2380 \ + initialCluster='' + index=0 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in "${ETCD_HOSTS[@]}" + do + # char '/' need to escape '\\/' + initialCluster="${initialCluster}etcdnode${index}=http:\\/\\/${item}:2380" + if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then + initialCluster=${initialCluster}"," + fi + index=$((index+1)) + done + #echo "initialCluster=${initialCluster}" + sed -i "s/INITIAL_CLUSTER_REPLACE/${initialCluster}/g" "$INSTALL_TEMP_DIR/etcd/etcd.service" + + cp "$INSTALL_TEMP_DIR/etcd/etcd.service" /etc/systemd/system/ +} + +## @description install etcd +## @audience public +## @stability stable +function install_etcd() +{ + index=$(indexByEtcdHosts "${LOCAL_HOST_IP}") + if [ -z "$index" ]; then + echo -e "STOP: This host\\033[31m[${LOCAL_HOST_IP}]\\033[0m is not in the ETCD server list\\033[31m[${ETCD_HOSTS[*]}]\\033[0m" + return 1 + fi + + install_etcd_bin + + install_etcd_config + + systemctl daemon-reload + systemctl enable etcd.service +} + +## @description uninstall etcd +## @audience public +## @stability stable +function uninstall_etcd() +{ + echo "stop etcd.service" + systemctl stop etcd.service + + echo "rm etcd ..." + rm /usr/bin/etcd + rm /usr/bin/etcdctl + rm -rf /var/lib/etcd + rm /etc/systemd/system/etcd.service + + systemctl daemon-reload +} + +## @description start etcd +## @audience public +## @stability stable +function start_etcd() +{ + systemctl restart etcd.service + + echo " ===== Check the status of the etcd service =====" + echo " exec etcdctl cluster-health" + etcdctl cluster-health + echo " exec etcdctl cluster-health" + etcdctl member list +} + +## @description stop etcd +## @audience public +## @stability stable +function stop_etcd() +{ + systemctl stop etcd.service +} diff --git a/dev-support/submarine-installer/scripts/menu.sh b/dev-support/submarine-installer/scripts/menu.sh new file mode 100644 index 0000000000000000000000000000000000000000..aff9d7a67d3446f02cee1ed0c6f4034e5979c3ac --- /dev/null +++ b/dev-support/submarine-installer/scripts/menu.sh @@ -0,0 +1,561 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description main menu +## @audience public +## @stability stable +main_menu() +{ +cat< [prepare system environment] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \\e[32m1.prepare operation system\\e[0m" +echo -e " \\e[32m2.prepare operation system kernel\\e[0m" +echo -e " \\e[32m3.prepare GCC version\\e[0m" +echo -e " \\e[32m4.check GPU\\e[0m" +echo -e " \\e[32m5.prepare user&group\\e[0m" +echo -e " \\e[32m6.prepare nvidia environment\\e[0m" +echo -e "" +echo -e " \\e[32mb.back main menu\\e[0m" +cat< [install component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \\e[32m1.install etcd\\e[0m" +echo -e " \\e[32m2.install docker\\e[0m" +echo -e " \\e[32m3.install calico network\\e[0m" +echo -e " \\e[32m4.install nvidia driver\\e[0m" +echo -e " \\e[32m5.install nvidia docker\\e[0m" +echo -e " \\e[32m6.install submarine autorun script\\e[0m" +echo -e " \\e[32m7.install yarn\\e[0m" +echo -e "" +echo -e " \\e[32mb.back main menu\\e[0m" +cat< [uninstall component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \\e[32m1.uninstall etcd\\e[0m" +echo -e " \\e[32m2.uninstall docker\\e[0m" +echo -e " \\e[32m3.uninstall calico network\\e[0m" +echo -e " \\e[32m4.uninstall nvidia driver\\e[0m" +echo -e " \\e[32m5.uninstall nvidia docker\\e[0m" +echo -e " \\e[32m6.uninstall submarine autorun script\\e[0m" +echo -e " \\e[32m7.uninstall yarn\\e[0m" +echo -e "" +echo -e " \\e[32mb.back main menu\\e[0m" +cat< [start component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \\e[32m1.start etcd\\e[0m" +echo -e " \\e[32m2.start docker\\e[0m" +echo -e " \\e[32m3.start calico network\\e[0m" +echo -e " \\e[32m4.start yarn\\e[0m" +echo -e "" +echo -e " \\e[32mb.back main menu\\e[0m" +cat< [stop component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \\e[32m1.stop etcd\\e[0m" +echo -e " \\e[32m2.stop docker\\e[0m" +echo -e " \\e[32m3.stop calico network\\e[0m" +echo -e " \\e[32m4.stop yarn\\e[0m" +echo -e "" +echo -e " \\e[32mb.back main menu\\e[0m" +cat< [install yarn] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \\e[32m1.install YARN [every host]\\e[0m" +echo -e " \\e[32m2.install YARN container-executor [every host]\\e[0m" +echo -e " \\e[32m3.install YARN Timeline [one host]\\e[0m" +echo -e "" +echo -e " \\e[32mb.back main menu\\e[0m" +cat< [install yarn] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \\e[32m1.uninstall Yarn\\e[0m" +echo -e "" +echo -e " \\e[32mb.back main menu\\e[0m" +cat< Failed to download nvidia-docker. + Please specify nvidia component for download_and_uncompress_nvidia_repo \\033[0m" + return 1 + fi + local component=$1 + + if [[ -d "${DOWNLOAD_DIR}/nvidia-docker-repo/${component}/centos7" ]]; then + echo "${DOWNLOAD_DIR}/nvidia-docker-repo/${component}/centos7 already exists." + else + # Trim the last slash of NVIDIA_DOCKER_GIT_SNAPSHOT_URL + local NVIDIA_DOCKER_URL="$(echo -e "${NVIDIA_DOCKER_GIT_SNAPSHOT_URL}" | sed -e 's/\/*$//')" + wget ${NVIDIA_DOCKER_URL}/${component}/tarball/gh-pages -O - | \ + tar -zx --strip-components=1 -C ${DOWNLOAD_DIR}/nvidia-docker-repo/${component} + if [[ $? -ne 0 ]]; then + echo -e "\\033[32mshell:> Failed to download ${component} of nvidia-docker + from ${NVIDIA_DOCKER_URL}/${component}/tarball/gh-pages \\033[0m" + fi + fi +} + +## @description install nvidia docker +## @audience public +## @stability stable +function install_nvidia_docker() +{ + # Backup /etc/docker/daemon.json + local DOCKER_DAEMON_BAK="${DOWNLOAD_DIR}/docker-daemon-bak" + if [[ ! -d "${DOCKER_DAEMON_BAK}" ]]; then + mkdir -p "${DOCKER_DAEMON_BAK}" + fi + cp /etc/docker/daemon.json "${DOCKER_DAEMON_BAK}" + echo "Backup /etc/docker/daemon.json in ${DOCKER_DAEMON_BAK}" + + # Remove nvidia docker 1.0 + remove_nvidia_docker_1.0 + + # Get nvidia-docker repo + if [[ ! -d "${DOWNLOAD_DIR}/nvidia-docker-repo" ]]; then + mkdir -p "${DOWNLOAD_DIR}/nvidia-docker-repo" + fi + local dockerRepo="${DOWNLOAD_DIR}/nvidia-docker-repo/nvidia-docker.repo" + if [[ -n "$DOWNLOAD_HTTP" ]]; then + wget -P "${DOWNLOAD_DIR}/nvidia-docker-repo/" \ + "${DOWNLOAD_HTTP}/downloads/nvidia-docker-repo/nvidia-docker/centos7/nvidia-docker.repo" + local DOWNLOAD_HTTP_REGEX=$(echo ${DOWNLOAD_HTTP} | sed 's/\//\\\//g') + echo "DOWNLOAD_HTTP_REGEX: ${DOWNLOAD_HTTP_REGEX}" + sed -i "s/https:\/\/nvidia.github.io/${DOWNLOAD_HTTP_REGEX}\/downloads\/nvidia-docker-repo/g" \ + "${dockerRepo}" + else + download_nvidia_docker_bin + local DOWNLOAD_DIR_REGEX=$(echo "${DOWNLOAD_DIR}" | sed 's/\//\\\//g') + cp "${DOWNLOAD_DIR}/nvidia-docker-repo/nvidia-docker/centos7/nvidia-docker.repo" \ + "${dockerRepo}" + sed -i "s/https:\/\/nvidia.github.io/file:\/\/${DOWNLOAD_DIR_REGEX}\/nvidia-docker-repo/g" \ + "${dockerRepo}" + fi + + # Install nvidia-docker + sudo cp ${dockerRepo} /etc/yum.repos.d/nvidia-docker.repo + echo -e "\\033[31m Installing nvidia-docker2 ...\\033[0m" + sudo yum install -y nvidia-docker2-${NVIDIA_DOCKER_VERSION}-1.docker${DOCKER_VERSION_NUM} + + # As nvidia-docker would overwrite daemon.json, append old daemon.json into the now daemon.json + COMBINE_JSON="${SCRIPTS_DIR}/combine-docker-daemons.py" + IS_NEW_JSON=$(python ${COMBINE_JSON} ${DOCKER_DAEMON_BAK}/daemon.json /etc/docker/daemon.json ${DOCKER_DAEMON_BAK}/daemon-new.json) + if [[ "${IS_NEW_JSON}" = "True" ]]; then + sudo cp ${DOCKER_DAEMON_BAK}/daemon-new.json /etc/docker/daemon.json + echo "Succeed to update /etc/docker/daemon.json" + else + echo "WARNING: /etc/docker/daemon.json is overrided by nvidia-docker and + can't be merged with the old daemon.json. Please update it manually + later." + fi + + # create nvidia driver library path + if [ ! -d "/var/lib/nvidia-docker/volumes/nvidia_driver" ]; then + echo "WARN: /var/lib/nvidia-docker/volumes/nvidia_driver folder path is not exist!" + sudo mkdir -p /var/lib/nvidia-docker/volumes/nvidia_driver + fi + + local nvidiaVersion + nvidiaVersion=$(get_nvidia_version) + echo -e "\\033[31m nvidia detect version is ${nvidiaVersion}\\033[0m" + + sudo mkdir "/var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}" + sudo mkdir "/var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin" + sudo mkdir "/var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64" + + sudo cp /usr/bin/nvidia* "/var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin" + sudo cp /usr/lib64/libcuda* "/var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64" + sudo cp /usr/lib64/libnvidia* "/var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64" + + echo -e "\\033[32m===== Please manually execute the following command =====\\033[0m" + echo -e "\\033[32mshell:> nvidia-docker run --rm ${DOCKER_REGISTRY}/nvidia/cuda:9.0-devel nvidia-smi +# If you don't see the list of graphics cards above, the NVIDIA driver installation failed. ===== +\\033[0m" + + echo -e "\\033[32m===== Please manually execute the following command =====\\033[0m" + echo -e "\\033[32m# Test with tf.test.is_gpu_available() +shell:> nvidia-docker run -it ${DOCKER_REGISTRY}/tensorflow/tensorflow:1.9.0-gpu bash +# In docker container +container:> python +python:> import tensorflow as tf +python:> tf.test.is_gpu_available() +python:> exit() +\\033[0m" +} + +## @description uninstall nvidia docker +## @audience public +## @stability stable +function uninstall_nvidia_docker() +{ + sudo yum remove -y nvidia-docker2-${NVIDIA_DOCKER_VERSION}-1.docker${DOCKER_VERSION_NUM} +} + +## @description uninstall nvidia docker 1.0 +## @audience public +## @stability stable +function remove_nvidia_docker_1.0() +{ + docker volume ls -q -f driver=nvidia-docker | \ + xargs -r -I{} -n1 docker ps -q -a -f volume={} | xargs -r docker rm -f + sudo yum remove nvidia-docker +} diff --git a/dev-support/submarine-installer/scripts/nvidia.sh b/dev-support/submarine-installer/scripts/nvidia.sh new file mode 100644 index 0000000000000000000000000000000000000000..a335214eacea1694aa1b87df22d9f5d71245981d --- /dev/null +++ b/dev-support/submarine-installer/scripts/nvidia.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nvidia_run_file="" + +## @description get nvidia version +## @audience public +## @stability stable +function get_nvidia_version() +{ + local nvidia_detect_info + nvidia_detect_info=$("nvidia-detect" -v) + echo $nvidia_detect_info | sed "s/^.*This device requires the current \([0-9.]*\).*/\1/" +} + +## @description download nvidia detect +## @audience public +## @stability stable +function download_nvidia_detect_rpm() +{ + if [[ -f ${DOWNLOAD_DIR}/nvidia/nvidia-detect.rpm ]]; then + echo "NVIDIA nvidia-detect.rpm already exists ${DOWNLOAD_DIR}/nvidia/ directory." + echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia-detect.rpm file is complete and can be used normally. =====" + elif [[ -n "$DOWNLOAD_HTTP" ]]; then + # Get nvidia-detect.rpm from download server + echo "Download ${MY_NVIDIA_DETECT_URL} ..." + wget -P "${DOWNLOAD_DIR}/nvidia/" "${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect.rpm" + if [[ $? -ne 0 ]]; then + echo -e "\\033[32mshell:> Failed to download nvidia-detect.rpm of nvidia + detect from ${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect.rpm \\033[0m" + fi + else + nvidiaDetectRpm=`curl --silent ${NVIDIA_DETECT_URL} | grep nvidia-detect- | sed 's/.*\(nvidia-detect-.*.rpm\).*/\1/g' | head -n 1` + # Trim the last slash of DOCKER_REPO + NVIDIA_DETECT_URL_TRIMED="$(echo -e "${NVIDIA_DETECT_URL}" | sed -e 's/\/*$//')" + wget --output-document="${DOWNLOAD_DIR}/nvidia/nvidia-detect.rpm" "${NVIDIA_DETECT_URL_TRIMED}/${nvidiaDetectRpm}" + if [[ $? -ne 0 ]]; then + echo -e "\\033[32mshell:> Failed to download nvidia-detect.rpm of nvidia + detect from ${NVIDIA_DETECT_URL}/${nvidiaDetectRpm} \\033[0m" + fi + fi +} + +## @description download nvidia driver +## @audience public +## @stability stable +function download_nvidia_driver() +{ + download_nvidia_detect_rpm + + # Install nvidia detect if needed + if ! [ -x "$(command -v nvidia-detect)" ]; then + echo 'Installing nvidia-detect ...' + sudo rpm -ivh "${DOWNLOAD_DIR}/nvidia/nvidia-detect.rpm" + fi + + echo "Check the gpu cards with nvidia-detect ..." + local nvidiaVersion + if [[ ${IS_DOWNLOAD_SERVER} && -n ${NVIDIA_DRIVER_VERSION} ]]; then + nvidiaVersion=${NVIDIA_DRIVER_VERSION} + else + nvidiaVersion=$(get_nvidia_version) + fi + echo -e "detect nvidia version is \\033[31m${nvidiaVersion}\\033[0m" + + # download NVIDIA driver + if [[ "$nvidiaVersion" = "" ]]; then + echo -e "\\033[31mERROR: No graphics card device detected.\\033[0m" + return 1 + else + nvidia_run_file="NVIDIA-Linux-x86_64-${nvidiaVersion}.run" + + # submarine http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_NVIDIA_DRIVER_RUN_URL="${DOWNLOAD_HTTP}/downloads/nvidia/${nvidia_run_file}" + else + # http://us.download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run + MY_NVIDIA_DRIVER_RUN_URL="http://us.download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/${nvidia_run_file}" + fi + + if [[ -f ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} ]]; then + echo "NVIDIA driver files already exist in the ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} directory." + echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia/${nvidia_run_file} file is complete and can be used normally. =====" + else + echo "Download the NVIDIA driver from the ${MY_NVIDIA_DRIVER_RUN_URL}" + wget -P "${DOWNLOAD_DIR}/nvidia/" "${MY_NVIDIA_DRIVER_RUN_URL}" + fi + fi +} + +## @description install nvidia +## @audience public +## @stability stable +function install_nvidia() +{ + download_nvidia_driver + + # Confirm that the system disables nouveau + local disable_nouveau_info + disable_nouveau_info=$(lsmod | grep nouveau) + if [[ "$disable_nouveau_info" = "" ]]; then + echo "===== Start installing the NVIDIA driver =====" + echo -e "Some options during the installation + Would you like to register the kernel module sources with DKMS? + This will allow DKMS to automatically build a new module, if you install a different kernel later. + \\033[33m[Yes]\\033[0m + Install NVIDIA's 32-bit compatibility libraries \\033[33m[Yes]\\033[0m + centos Install NVIDIA's 32-bit compatibility libraries + \\033[33m[Yes]\\033[0m + Would you like to run the nvidia-xconfig utility to automatically update your X configuration file... + \\033[33m[No]\\033[0m" + sleep 2 + sh "${DOWNLOAD_DIR}/nvidia/${nvidia_run_file}" + else + echo -e "ERROR: Nouveau is not disabled" + return 1 + fi + + echo -e "\\033[32m===== execute nvidia-smi. You should be able to see the list of graphics cards =====\\033[0m" + sleep 1 + nvidia-smi +} + +## @description uninstall nvidia +## @audience public +## @stability stable +function uninstall_nvidia() +{ + if [ -x "$(command -v nvidia-detect)" ]; then + sudo yum remove nvidia-detect + echo "Succeeded to remove nvidia-detect" + fi + + if [ ! -f "/usr/bin/nvidia-uninstall" ]; then + echo -e "\\033[31mERROR: /usr/bin/nvidia-uninstall file is not exist!\\033[0m" + return 1 + fi + + echo -e "execute /usr/bin/nvidia-uninstall" + sudo /usr/bin/nvidia-uninstall +} diff --git a/dev-support/submarine-installer/scripts/submarine.sh b/dev-support/submarine-installer/scripts/submarine.sh new file mode 100644 index 0000000000000000000000000000000000000000..f3baec2e90d8752a342b90b6718444c201bf8bf8 --- /dev/null +++ b/dev-support/submarine-installer/scripts/submarine.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description install submarine autorun script +## @audience public +## @stability stable +function install_submarine() +{ + cp "${PACKAGE_DIR}/submarine/submarine.sh" /etc/rc.d/init.d/submarine.sh + chmod +x /etc/rc.d/init.d/submarine.sh + chmod +x /etc/rc.d/rc.local + + if [ "$(grep -c "/etc/rc.d/init.d/submarine.sh" /etc/rc.d/rc.local)" -eq '0' ]; then + echo "/etc/rc.d/init.d/submarine.sh">> /etc/rc.d/rc.local + fi +} + +## @description uninstall submarine autorun script +## @audience public +## @stability stable +function uninstall_submarine() +{ + rm /etc/rc.d/init.d/submarine.sh +} \ No newline at end of file diff --git a/dev-support/submarine-installer/scripts/utils.sh b/dev-support/submarine-installer/scripts/utils.sh new file mode 100644 index 0000000000000000000000000000000000000000..b3ecbcb371220a0228aea845f59133c89a9a000e --- /dev/null +++ b/dev-support/submarine-installer/scripts/utils.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description check install user +## @audience public +## @stability stable +function check_install_user() +{ + if [[ $(id -u) -ne 0 ]];then + echo "This script must be run with a ROOT user!" + exit # don't call exit_install() + fi +} + +## @description exit install +## @audience public +## @stability stable +function exit_install() +{ + echo "Exit the installation!" + exit +} + +## @description Check if the IP address format is correct +## @audience public +## @stability stable +function valid_ip() +{ + local ip=$1 + local stat=1 + + if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then + OIFS=$IFS + IFS='.' + ip=($ip) + IFS=$OIFS + + if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then + stat=$? + fi + fi + + return $stat +} + +## @description Check if the configuration file configuration is correct +## @audience public +## @stability stable +function check_install_conf() +{ + echo "Check if the configuration file configuration is correct ..." + + # check YARN_SECURITY's value + if [ -z "${YARN_SECURITY}" ]; then + echo "YARN_SECURITY=[$YARN_SECURITY] is empty! please use true or false " + exit_install + fi + + # check resource manager + rmCount=${#YARN_RESOURCE_MANAGER_HOSTS[@]} + if [[ $rmCount -gt 2 ]]; then # <>2 + echo "Number of resource manager nodes = [$rmCount], must be configured equal 2 servers! " + exit_install + fi + + # + if [ -z "${YARN_REGISTRY_DNS_HOST_PORT}" ]; then + echo "YARN_REGISTRY_DNS_HOST_PORT=[$YARN_REGISTRY_DNS_HOST_PORT] is empty! " + exit_install + fi + + # Check if it is empty + if [[ "${YARN_SECURITY}" = "true" && -z "${LOCAL_REALM}" ]]; then + echo "LOCAL_REALM=[$LOCAL_REALM] can not be empty! " + exit_install + fi + + if [[ "${YARN_SECURITY}" = "true" && -z "${HADOOP_KEYTAB_LOCATION}" ]]; then + echo "HADOOP_KEYTAB_LOCATION=[$HADOOP_KEYTAB_LOCATION] can not be empty! " + exit_install + fi + + if [[ "${YARN_SECURITY}" = "true" && -z "${HADOOP_PRINCIPAL}" ]]; then + echo "HADOOP_PRINCIPAL=[$HADOOP_PRINCIPAL] can not be empty! " + exit_install + fi + + if [[ "${YARN_SECURITY}" = "true" && -z "${MAPRED_KEYTAB_LOCATION}" ]]; then + echo "MAPRED_KEYTAB_LOCATION=[$MAPRED_KEYTAB_LOCATION] can not be empty! " + exit_install + fi + + if [[ "${YARN_SECURITY}" = "true" && -z "${YARN_KEYTAB_LOCATION}" ]]; then + echo "YARN_KEYTAB_LOCATION=[$YARN_KEYTAB_LOCATION] can not be empty! " + exit_install + fi + + if [[ "${YARN_SECURITY}" = "true" && -z "${HTTP_KEYTAB_LOCATION}" ]]; then + echo "HTTP_KEYTAB_LOCATION=[$HTTP_KEYTAB_LOCATION] can not be empty! " + exit_install + fi + + # check etcd conf + hostCount=${#ETCD_HOSTS[@]} + if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 + echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " + exit_install + fi + echo "Check if the configuration file configuration is correct [ Done ]" +} + +## @description index by EtcdHosts list +## @audience public +## @stability stable +function indexByEtcdHosts() { + index=0 + while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do + if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then + echo $index + return + fi + (( index++ )) + done + echo "" +} + +## @description index by Resource Manager Hosts list +## @audience public +## @stability stable +function indexByRMHosts() { + index=0 + while [ "$index" -lt "${#YARN_RESOURCE_MANAGER_HOSTS[@]}" ]; do + if [ "${YARN_RESOURCE_MANAGER_HOSTS[$index]}" = "$1" ]; then + echo $index + return + fi + (( index++ )) + done + echo "" +} + +## @description index of node manager exclude Hosts list +## @audience public +## @stability stable +function indexOfNMExcludeHosts() { + index=0 + while [ "$index" -lt "${#YARN_NODE_MANAGER_EXCLUDE_HOSTS[@]}" ]; do + if [ "${YARN_NODE_MANAGER_EXCLUDE_HOSTS[$index]}" = "$1" ]; then + echo $index + return + fi + (( index++ )) + done + echo "" +} + +## @description index by Resource Manager Hosts list +## @audience public +## @stability stable +function pathExitsOnHDFS() { + exists=$("${HADOOP_HOME}/bin/hadoop" dfs -ls -d "$1") + echo "${exists}" +} + +## @description get local IP +## @audience public +## @stability stable +function getLocalIP() +{ + local _ip _myip _line _nl=$'\n' + while IFS=$': \t' read -r -a _line ;do + [ -z "${_line%inet}" ] && + _ip=${_line[${#_line[1]}>4?1:2]} && + [ "${_ip#127.0.0.1}" ] && _myip=$_ip + done< <(LANG=C /sbin/ifconfig) + printf "%s" ${1+-v} "$1" "%s${_nl:0:$((${#1}>0?0:1))}" "$_myip" +} + +## @description get ip list +## @audience public +## @stability stable +function get_ip_list() +{ + array=$(ip -o -4 addr | awk '{print $4}' | grep -v 127 | cut -d/ -f1) + + for ip in "${array[@]}" + do + LOCAL_HOST_IP_LIST+=(${ip}) + done +} diff --git a/dev-support/submarine-installer/scripts/xmlcombine.py b/dev-support/submarine-installer/scripts/xmlcombine.py new file mode 100644 index 0000000000000000000000000000000000000000..2b049d02069e6c9f853207c253e9aa28e93d43f9 --- /dev/null +++ b/dev-support/submarine-installer/scripts/xmlcombine.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from xml.etree import ElementTree + +def run(files): + first = None + for filename in files: + data = ElementTree.parse(filename).getroot() + if first is None: + first = data + else: + first.extend(data) + if first is not None: + print ElementTree.tostring(first) + +if __name__ == "__main__": + run(sys.argv[1:]) diff --git a/dev-support/submarine-installer/scripts/yarn.sh b/dev-support/submarine-installer/scripts/yarn.sh new file mode 100644 index 0000000000000000000000000000000000000000..461c20e3ab027effaa17c0878cd77537e6dcb338 --- /dev/null +++ b/dev-support/submarine-installer/scripts/yarn.sh @@ -0,0 +1,640 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description install yarn +## @audience public +## @stability stable +function install_yarn() +{ + # base YARN_SECURITY determine which etc is + rm -f ${PACKAGE_DIR}/hadoop/yarn/etc + ln -s ${PACKAGE_DIR}/hadoop/yarn/etc_secure ${PACKAGE_DIR}/hadoop/yarn/etc + + initialize_temp + + host=$(hostname) + index=$(indexByRMHosts "${host}") + if [[ -n "$index" || "x$YARN_TIMELINE_HOST" != "x$host" ]]; then + kinit -kt ${HADOOP_KEYTAB_LOCATION} ${HADOOP_PRINCIPAL} + fi + + install_java_tarball + if [[ $? = 1 ]]; then + return 1 + fi + install_yarn_tarball + if [[ $? = 1 ]]; then + return 1 + fi + install_yarn_sbin + install_yarn_rm_nm + install_yarn_service + install_registry_dns + install_timeline_server + install_job_history + install_mapred + install_spark_shuffle + install_lzo_native + + # copy file + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/hdfs-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/capacity-scheduler.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/resource-types.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/log4j.properties" "${HADOOP_HOME}/etc/hadoop/" + chown "${HADOOP_SETUP_USER}":yarn "${HADOOP_HOME}"/etc/hadoop/* +} + +## @description Initialize tmp dir for installation. +## @audience public +## @stability stable +function initialize_temp() +{ + mkdir -p "${INSTALL_TEMP_DIR}/hadoop" + \cp -rf "${PACKAGE_DIR}/hadoop/yarn" "${INSTALL_TEMP_DIR}/hadoop" + isGpuEnabled=$(nvidia-smi) + if [[ -n "$isGpuEnabled" ]]; then + python ${SCRIPTS_DIR}/xmlcombine.py ${PACKAGE_DIR}/hadoop/yarn/etc/hadoop/yarn-site.xml ${PACKAGE_DIR}/hadoop/yarn/etc/hadoop/gpu/yarn-site-gpu.xml > "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/yarn-site.xml" + fi + chown -R ${HADOOP_SETUP_USER} "${INSTALL_TEMP_DIR}/hadoop" +} + +## @description uninstall yarn +## @audience public +## @stability stable +function uninstall_yarn() +{ + executor_dir=$(dirname "${YARN_CONTAINER_EXECUTOR_PATH}") + executor_conf_dir=$(dirname "${executor_dir}")/etc/hadoop + + rm -rf "${YARN_CONTAINER_EXECUTOR_PATH}" + rm -rf "${executor_conf_dir}" + rm -rf "${HADOOP_HOME}" +} + +## @description install yarn container executor +## @audience public +## @stability stable +function install_yarn_container_executor() +{ + echo "install yarn container executor file ..." + + executor_dir=$(dirname "${YARN_CONTAINER_EXECUTOR_PATH}") + if [ ! -d "${executor_dir}" ]; then + mkdir -p "${executor_dir}" + fi + if [ -f "${YARN_CONTAINER_EXECUTOR_PATH}" ]; then + if [ -f "${HADOOP_HOME}/bin/container-executor" ]; then + rm ${YARN_CONTAINER_EXECUTOR_PATH} + fi + fi + + if [ -f "${HADOOP_HOME}/bin/container-executor" ]; then + cp -f "${HADOOP_HOME}/bin/container-executor" "${YARN_CONTAINER_EXECUTOR_PATH}" + rm "${HADOOP_HOME}/bin/container-executor" + fi + + sudo chmod 6755 "${executor_dir}" + sudo chown :yarn "${YARN_CONTAINER_EXECUTOR_PATH}" + sudo chmod 6050 "${YARN_CONTAINER_EXECUTOR_PATH}" +} + +## @description Deploy hadoop yarn tar ball +## @audience public +## @stability stable +function install_yarn_tarball() +{ + tag=`date '+%Y%m%d%H%M%S'` + if [ -f "${PACKAGE_DIR}/hadoop/${HADOOP_TARBALL}" ]; then + tar -zxvf "${PACKAGE_DIR}/hadoop/${HADOOP_TARBALL}" -C "${PACKAGE_DIR}/hadoop/" + mv "${PACKAGE_DIR}/hadoop/${HADOOP_VERSION}" "/home/${HADOOP_SETUP_USER}/${HADOOP_VERSION}-${tag}" + chown -R ${HADOOP_SETUP_USER} "/home/hadoop/${HADOOP_VERSION}-${tag}" + if [[ -d "${HADOOP_HOME}" ]] || [[ -L "${HADOOP_HOME}" ]]; then + rm -rf ${HADOOP_HOME} + fi + ln -s "/home/hadoop/${HADOOP_VERSION}-${tag}" "${HADOOP_HOME}" + chown ${HADOOP_SETUP_USER} "${HADOOP_HOME}" + else + echo "ERROR: Please put ${HADOOP_TARBALL} in the path of ${PACKAGE_DIR}/hadoop/ firstly." + return 1 + fi +} + +## @description Deploy java tar ball +## @audience public +## @stability stable +function install_java_tarball() +{ + if [[ -d "${JAVA_HOME}" ]] || [[ -L "${JAVA_HOME}" ]]; then + echo "JAVA_HOME already exists. There is no need to install java." + else + if [[ -f "${PACKAGE_DIR}/java/${JAVA_TARBALL}" ]]; then + tar -zxvf "${PACKAGE_DIR}/java/${JAVA_TARBALL}" -C "${PACKAGE_DIR}/java/" + mv "${PACKAGE_DIR}/java/${JAVA_VERSION}" "/home/${HADOOP_SETUP_USER}/${JAVA_VERSION}" + chown -R ${HADOOP_SETUP_USER} "/home/hadoop/${JAVA_VERSION}" + ln -s "/home/hadoop/${JAVA_VERSION}" "${JAVA_HOME}" + else + echo "Error: Failed to install java, please put java tarball in the path of + ${PACKAGE_DIR}/java/${JAVA_TARBALL}" + return 1 + fi + fi +} + +## @description install yarn resource & node manager +## @audience public +## @stability stable +function install_yarn_rm_nm() +{ + echo "install yarn config file ..." + host=$(hostname) + + find="/" + replace="\\/" + escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace} + escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace} + escape_yarn_keytab_location=${YARN_KEYTAB_LOCATION//$find/$replace} + escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace} + escape_http_keytab_location=${HTTP_KEYTAB_LOCATION//$find/$replace} + escape_yarn_nodemanager_nodes_exclude_path=${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH//$find/$replace} + escape_yarn_nodemanager_recovery_dir=${YARN_NODEMANAGER_RECOVERY_DIR//$find/$replace} + escape_fs_defaults=${FS_DEFAULTFS//$find/$replace} + escape_hadoop_http_authentication_signature_secret_file=${HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE//$find/$replace} + escape_hadoop_home=${HADOOP_HOME//$find/$replace} + + # container-executor.cfg` + sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS_REPLACE/${YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/HADOOP_HOME_REPLACE/${escape_hadoop_home}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + + # enable cgroup for yarn + . "${PACKAGE_DIR}/submarine/submarine.sh" + + # Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error. + sed -i '1,16d' "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + + executor_dir=$(dirname "${YARN_CONTAINER_EXECUTOR_PATH}") + executor_conf_dir=$(dirname "${executor_dir}")/etc/hadoop + if [ ! -d "${executor_conf_dir}" ]; then + sudo mkdir -p "${executor_conf_dir}" + fi + + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" "${executor_conf_dir}" + + # yarn-site.xml + sed -i "s/YARN_RESOURCE_MANAGER_HOSTS1_REPLACE/${YARN_RESOURCE_MANAGER_HOSTS[0]}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_RESOURCE_MANAGER_HOSTS2_REPLACE/${YARN_RESOURCE_MANAGER_HOSTS[1]}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/HTTP_KEYTAB_LOCATION_REPLACE/${escape_http_keytab_location}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/LOCAL_CLUSTER_ID_REPLACE/${LOCAL_CLUSTER_ID}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + # Make nodemanager local dirs if the host is not in YARN_NODE_MANAGER_EXCLUDE_HOSTS + index=$(indexOfNMExcludeHosts "${host}") + if [ -z "$index" ]; then + arr=(${YARN_NODEMANAGER_LOCAL_DIRS//,/ }) + index=0 + while [ "$index" -lt "${#arr[@]}" ]; do + mkdir -p "${arr[$index]}" + (( index++ )) + done + + arr=(${YARN_NODEMANAGER_LOG_DIRS//,/ }) + index=0 + while [ "$index" -lt "${#arr[@]}" ]; do + mkdir -p "${arr[$index]}" + (( index++ )) + done + + arr=(${YARN_NODEMANAGER_LOCAL_HOME_PATHS//,/ }) + index=0 + while [ "$index" -lt "${#arr[@]}" ]; do + chown -R yarn "${arr[$index]}" + (( index++ )) + done + fi + + sed -i "s/YARN_ZK_ADDRESS_REPLACE/${YARN_ZK_ADDRESS}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_KEYTAB_LOCATION_REPLACE/${escape_yarn_keytab_location}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_DOCKER_CONTAINER_DEFAULT_NETWORK_REPLACE/${YARN_DOCKER_CONTAINER_DEFAULT_NETWORK}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS_REPLACE/${YARN_DOCKER_ALLOWED_CONTAINER_NETWORKS}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + + sed -i "s/YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH_REPLACE/${escape_yarn_nodemanager_nodes_exclude_path}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + node_exclude_dir=$(dirname "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}") + if [[ ! -d "${node_exclude_dir}" ]]; then + mkdir -p "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" + fi + if [[ ! -f "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" ]]; then + touch "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" + chmod 777 "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" + fi + + sed -i "s/YARN_NODEMANAGER_RECOVERY_DIR_REPLACE/${escape_yarn_nodemanager_recovery_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + mkdir -p "${YARN_NODEMANAGER_RECOVERY_DIR}" + chmod 777 "${YARN_NODEMANAGER_RECOVERY_DIR}" + + # core-site.xml + sed -i "s/LOCAL_REALM_REPLACE/${LOCAL_REALM}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + sed -i "s/YARN_ZK_ADDRESS_REPLACE/${YARN_ZK_ADDRESS}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + sed -i "s/FS_DEFAULTFS_REPLACE/${escape_fs_defaults}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + sed -i "s/HTTP_KEYTAB_LOCATION_REPLACE/${escape_http_keytab_location}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + + # WARN: ${HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE} Can not be empty! + echo 'hello submarine' > "${HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE}" + escape_hadoop_http_authentication_signature_secret_file=${HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE//$find/$replace} + sed -i "s/HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE_REPLACE/${escape_hadoop_http_authentication_signature_secret_file}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/hdfs-site.xml" "${HADOOP_HOME}/etc/hadoop/" + + install_yarn_container_executor +} + +function install_spark_shuffle() { + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/spark* "${HADOOP_HOME}/share/hadoop/yarn/lib/" +} + +function install_lzo_native() { + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/native/libgpl* "${HADOOP_HOME}/lib/native/" + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/yarn/lib/" + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/hdfs/lib/" + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/common/lib/" + if [ ! -d "${HADOOP_HOME}/share/hadoop/mapreduce/lib/" ]; then + mkdir -p "${HADOOP_HOME}/share/hadoop/mapreduce/lib/" + fi + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/mapreduce/lib/" +} + +function install_mapred() { + find="/" + replace="\\/" + escape_mapred_keytab_location=${MAPRED_KEYTAB_LOCATION//$find/$replace} + escape_yarn_app_mapreduce_am_staging_dir=${YARN_APP_MAPREDUCE_AM_STAGING_DIR//$find/$replace} + escape_fs_defaults=${FS_DEFAULTFS//$find/$replace} + + sed -i "s/FS_DEFAULTFS_REPLACE/${escape_fs_defaults}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" + sed -i "s/YARN_APP_MAPREDUCE_AM_STAGING_DIR_REPLACE/${escape_yarn_app_mapreduce_am_staging_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" + host=$(hostname) + index=$(indexByRMHosts "${host}") + if [[ -n "$index" ]]; then + # Only RM needs to execute the following code + pathExists=$(pathExitsOnHDFS "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}") + if [[ -z "${pathExists}" ]]; then + echo "Create hdfs path ${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + "${HADOOP_HOME}/bin/hadoop" dfs -mkdir -p "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + "${HADOOP_HOME}/bin/hadoop" dfs -chown yarn:hadoop "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + "${HADOOP_HOME}/bin/hadoop" dfs -chmod 1777 "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + fi + fi + sed -i "s/MAPRED_KEYTAB_LOCATION_REPLACE/${escape_mapred_keytab_location}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" +} + +function install_yarn_sbin() { + find="/" + replace="\\/" + escape_yarn_gc_log_dir=${YARN_GC_LOG_DIR//$find/$replace} + escape_java_home=${JAVA_HOME//$find/$replace} + escape_hadoop_home=${HADOOP_HOME//$find/$replace} + escape_yarn_pid_dir=${YARN_PID_DIR//$find/$replace} + escape_yarn_log_dir=${YARN_LOG_DIR//$find/$replace} + cp -R ${PACKAGE_DIR}/hadoop/yarn/sbin/* "${HADOOP_HOME}/sbin/" + chown "${HADOOP_SETUP_USER}":yarn "${HADOOP_HOME}"/sbin/* + + if [ ! -d "$YARN_GC_LOG_DIR" ]; then + mkdir -p "$YARN_GC_LOG_DIR" + chown "${HADOOP_SETUP_USER}":yarn "${YARN_GC_LOG_DIR}" + chmod 775 "${YARN_GC_LOG_DIR}" + fi + + if [ ! -d "$YARN_LOG_DIR" ]; then + mkdir -p "$YARN_LOG_DIR" + chown "${HADOOP_SETUP_USER}":yarn "${YARN_LOG_DIR}" + chmod 775 "${YARN_LOG_DIR}" + fi + + if [ ! -d "$YARN_PID_DIR" ]; then + mkdir -p "$YARN_PID_DIR" + chown "${HADOOP_SETUP_USER}":yarn "${YARN_PID_DIR}" + chmod 775 "${YARN_PID_DIR}" + fi + + sed -i "s/YARN_LOG_DIR_REPLACE/${escape_yarn_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/YARN_PID_DIR_REPLACE/${escape_yarn_pid_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/JAVA_HOME_REPLACE/${escape_java_home}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/HADOOP_HOME_REPLACE/${escape_hadoop_home}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/GC_LOG_DIR_REPLACE/${escape_yarn_gc_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + cp -R "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" "${HADOOP_HOME}/etc/hadoop/" + + sed -i "s/GC_LOG_DIR_REPLACE/${escape_yarn_gc_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/yarn-env.sh" + cp -R "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/yarn-env.sh" "${HADOOP_HOME}/etc/hadoop/" + + sed -i "s/GC_LOG_DIR_REPLACE/${escape_yarn_gc_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/mapred-env.sh" + cp -R "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/mapred-env.sh" "${HADOOP_HOME}/etc/hadoop/" + +cat< +hadoop.proxyuser.yarn.hosts +* + + +hadoop.proxyuser.yarn.groups +* + +----------------------------------------------------------------- +HELPINFO +} + +function install_registry_dns() { + sed -i "s/YARN_REGISTRY_DNS_HOST_REPLACE/${YARN_REGISTRY_DNS_HOST}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_REGISTRY_DNS_HOST_PORT_REPLACE/${YARN_REGISTRY_DNS_HOST_PORT}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" +} + +function install_job_history() { + sed -i "s/YARN_JOB_HISTORY_HOST_REPLACE/${YARN_JOB_HISTORY_HOST}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" +} + +## @description install yarn timeline server +## @audience public +## @stability stable +## http://hadoop.apache.org/docs/r3.1.0/hadoop-yarn/hadoop-yarn-site/TimelineServer.html +function install_timeline_server() +{ + find="/" + replace="\\/" + escape_aggregated_log_dir=${YARN_AGGREGATED_LOG_DIR//$find/$replace} + escape_yarn_timeline_service_hbase_configuration_file=${YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE//$find/$replace} + escape_yarn_keytab_location=${YARN_KEYTAB_LOCATION//$find/$replace} + escape_yarn_timeline_fs_store_dir=${YARN_TIMELINE_FS_STORE_DIR//$find/$replace} + # timeline v1.5 + escape_yarn_timeline_service_leveldb_state_store_path=${YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH//$find/$replace} + + # set leveldb configuration + sed -i "s/YARN_KEYTAB_LOCATION_REPLACE/${escape_yarn_keytab_location}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_AGGREGATED_LOG_DIR_REPLACE/${escape_aggregated_log_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE_REPLACE/${escape_yarn_timeline_service_hbase_configuration_file}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + # timeline v1.5 + sed -i "s/YARN_TIMELINE_HOST_REPLACE/${YARN_TIMELINE_HOST}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_REPLACE/${escape_yarn_timeline_service_leveldb_state_store_path}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_TIMELINE_FS_STORE_DIR_REPLACE/${escape_yarn_timeline_fs_store_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + + host=$(hostname) + if [ "x$YARN_TIMELINE_HOST" != "x$host" ]; then + return 0 + fi + + echo -n "Do you want to create hdfs directories for timelineserver[Y/N]?" + read -r answer + echo "$answer" + if [[ "$answer" = "y" || "$answer" = "Y" ]]; then + echo "Continue installing ..." + else + echo "Stop creating hdfs directories for timelineserver" + return 0 + fi + + echo "install yarn timeline server V1.5 ..." + +cat< grant 'yarn', 'RWC' +> grant 'HTTP', 'R' +HELPINFO + echo -n "Have you done the above operation[Y/N]?" + read -r answer + if [[ "$answer" = "y" || "$answer" = "Y" ]]; then + echo "Continue installing ..." + else + echo "Stop installing the timeline server V2" + return 0 + fi + + pathExists=$(pathExitsOnHDFS "/hbase") + if [[ -z "${pathExists}" ]]; then + "${HADOOP_HOME}/bin/hadoop" fs -mkdir -p "/hbase" + "${HADOOP_HOME}/bin/hadoop" fs -chmod -R 755 "/hbase" + fi + + pathExists=$(pathExitsOnHDFS "/hbase/coprocessor") + if [[ -z "${pathExists}" ]]; then + "${HADOOP_HOME}/bin/hadoop" fs -mkdir -p "/hbase/coprocessor" + "${HADOOP_HOME}/bin/hadoop" fs -chmod -R 755 "/hbase/coprocessor" + fi + + "${HADOOP_HOME}/bin/hadoop" fs -put "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-hbase-coprocessor-3.*.jar "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar" + "${HADOOP_HOME}/bin/hadoop" fs -chmod 755 "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar" + +cat</lib path: +HELPINFO + + if [[ -n "${HBASE_HOME}" ]]; then + cp "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-hbase-common-3.*-SNAPSHOT.jar "${HBASE_HOME}/lib" + cp "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-hbase-client-3.*-SNAPSHOT.jar "${HBASE_HOME}/lib" + cp "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-3.*-SNAPSHOT.jar "${HBASE_HOME}/lib" + fi + +cat< path, Execute the following command to create a schema +> bin/hbase org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator -create +HELPINFO + echo -n "Have you done the above operation[Y/N]?" + read -r answer + if [[ "$answer" = "y" || "$answer" = "Y" ]]; then + echo "Continue installing ..." + else + echo "Please initialize hbase timeline schema before you start timelineserver" + fi +} + +## @description start yarn +## @audience public +## @stability stable +function start_yarn() +{ + current_user=$(whoami) + host=$(hostname) + + # Start RM if the host is in YARN_RESOURCE_MANAGER_HOSTS + index=$(indexByRMHosts "${host}") + if [ -n "$index" ]; then + # Only RM needs to execute the following code + echo "Starting resourcemanager..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-resourcemanager.sh" + else + "${HADOOP_HOME}/sbin/start-resourcemanager.sh" + fi + fi + + # Start nodemanager + index=$(indexOfNMExcludeHosts "${host}") + if [ -z "$index" ]; then + echo "Starting nodemanager..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-nodemanager.sh" + else + "${HADOOP_HOME}/sbin/start-nodemanager.sh" + fi + fi + + # Start timeline + if [ "x$YARN_TIMELINE_HOST" = "x$host" ]; then + echo "Starting timelineserver..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-timelineserver.sh" + su - yarn -c "${HADOOP_HOME}/sbin/start-timelinereader.sh" + else + ${HADOOP_HOME}/sbin/start-timelineserver.sh + ${HADOOP_HOME}/sbin/start-timelinereader.sh + fi + fi + + # Start jobhistory + if [ "x$YARN_JOB_HISTORY_HOST" = "x$host" ]; then + echo "Starting mapreduce job history..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-mr-jobhistory.sh" + else + ${HADOOP_HOME}/sbin/start-mr-jobhistory.sh + fi + fi + + # Start registrydns + if [ "x$YARN_REGISTRY_DNS_HOST" = "x$host" ]; then + echo "Starting registry dns..." + sudo ${HADOOP_HOME}/sbin/start-registrydns.sh + fi +} + +## @description stop yarn +## @audience public +## @stability stable +function stop_yarn() +{ + current_user=$(whoami) + host=$(hostname) + # Stop RM if the host is in YARN_RESOURCE_MANAGER_HOSTS + index=$(indexByRMHosts "${host}") + if [ -n "$index" ]; then + # Only RM needs to execute the following code + if [ ${current_user} != "yarn" ]; then + echo "Stopping resourcemanager..." + su - yarn -c "${HADOOP_HOME}/sbin/stop-resourcemanager.sh" + else + "${HADOOP_HOME}/sbin/stop-resourcemanager.sh" + fi + fi + + # Stop nodemanager + index=$(indexOfNMExcludeHosts "${host}") + if [ -z "$index" ]; then + echo "Stopping nodemanager..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/stop-nodemanager.sh" + else + "${HADOOP_HOME}/sbin/stop-nodemanager.sh" + fi + fi + + # Stop timeline + if [ "x$YARN_TIMELINE_HOST" = "x$host" ]; then + echo "Stopping timelineserver..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/stop-timelineserver.sh" + su - yarn -c "${HADOOP_HOME}/sbin/stop-timelinereader.sh" + else + ${HADOOP_HOME}/sbin/stop-timelineserver.sh + ${HADOOP_HOME}/sbin/stop-timelinereader.sh + fi + fi + + # Stop jobhistory + if [ "x$YARN_JOB_HISTORY_HOST" = "x$host" ]; then + echo "Stopping mapreduce job history..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/stop-mr-jobhistory.sh" + else + ${HADOOP_HOME}/sbin/stop-mr-jobhistory.sh + fi + fi + + # Stop registrydns + if [ "x$YARN_REGISTRY_DNS_HOST" = "x$host" ]; then + echo "Stopping registry dns..." + sudo ${HADOOP_HOME}/sbin/stop-registrydns.sh + fi +} + diff --git a/dev-support/submarine-installer/scripts/yarn_insecure.sh b/dev-support/submarine-installer/scripts/yarn_insecure.sh new file mode 100644 index 0000000000000000000000000000000000000000..8e45710dc795075a3b1f572072600e5a32dd6bbd --- /dev/null +++ b/dev-support/submarine-installer/scripts/yarn_insecure.sh @@ -0,0 +1,626 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description install yarn +## @audience public +## @stability stable +function install_yarn_insecure() +{ + # change the soft link of etc + rm -f ${PACKAGE_DIR}/hadoop/yarn/etc + ln -s ${PACKAGE_DIR}/hadoop/yarn/etc_insecure ${PACKAGE_DIR}/hadoop/yarn/etc + + initialize_temp_insecure + + host=$(hostname) + index=$(indexByRMHosts "${host}") + #if [[ -n "$index" || "x$YARN_TIMELINE_HOST" != "x$host" ]]; then + # kinit -kt ${HADOOP_KEYTAB_LOCATION} ${HADOOP_PRINCIPAL} + #fi + + install_java_tarball + if [[ $? = 1 ]]; then + return 1 + fi + install_yarn_tarball_insecure + if [[ $? = 1 ]]; then + return 1 + fi + install_yarn_sbin_insecure + install_yarn_rm_nm_insecure + install_yarn_service_insecure + install_registry_dns_insecure + install_timeline_server_insecure + install_job_history_insecure + install_mapred_insecure + install_spark_shuffle_insecure + install_lzo_native_insecure + + # copy file + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/hdfs-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/capacity-scheduler.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/resource-types.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/log4j.properties" "${HADOOP_HOME}/etc/hadoop/" + chown "${HADOOP_SETUP_USER}":yarn "${HADOOP_HOME}"/etc/hadoop/* +} + +## @description Initialize tmp dir for installation. +## @audience public +## @stability stable +function initialize_temp_insecure() +{ + mkdir -p "${INSTALL_TEMP_DIR}/hadoop" + \cp -rf "${PACKAGE_DIR}/hadoop/yarn" "${INSTALL_TEMP_DIR}/hadoop/" + isGpuEnabled=$(nvidia-smi) + if [[ -n "$isGpuEnabled" ]]; then + python ${SCRIPTS_DIR}/xmlcombine.py ${PACKAGE_DIR}/hadoop/yarn/etc/hadoop/yarn-site.xml ${PACKAGE_DIR}/hadoop/yarn/etc/hadoop/gpu/yarn-site-gpu.xml > "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/yarn-site.xml" + fi + chown -R ${HADOOP_SETUP_USER} "${INSTALL_TEMP_DIR}/hadoop" +} + +## @description uninstall yarn +## @audience public +## @stability stable +function uninstall_yarn() +{ + executor_dir=$(dirname "${YARN_CONTAINER_EXECUTOR_PATH}") + executor_conf_dir=$(dirname "${executor_dir}")/etc/hadoop + + rm -rf "${YARN_CONTAINER_EXECUTOR_PATH}" + rm -rf "${executor_conf_dir}" + rm -rf "${HADOOP_HOME}" +} + +## @description install yarn container executor +## @audience public +## @stability stable +function install_yarn_container_executor_insecure() +{ + echo "install yarn container executor file ..." + + executor_dir=$(dirname "${YARN_CONTAINER_EXECUTOR_PATH}") + if [ ! -d "${executor_dir}" ]; then + mkdir -p "${executor_dir}" + fi + if [ -f "${YARN_CONTAINER_EXECUTOR_PATH}" ]; then + if [ -f "${HADOOP_HOME}/bin/container-executor" ]; then + rm ${YARN_CONTAINER_EXECUTOR_PATH} + fi + fi + + if [ -f "${HADOOP_HOME}/bin/container-executor" ]; then + cp -f "${HADOOP_HOME}/bin/container-executor" "${YARN_CONTAINER_EXECUTOR_PATH}" + rm "${HADOOP_HOME}/bin/container-executor" + fi + + sudo chmod 6755 "${executor_dir}" + sudo chown :yarn "${YARN_CONTAINER_EXECUTOR_PATH}" + sudo chmod 6050 "${YARN_CONTAINER_EXECUTOR_PATH}" +} + +## @description Deploy hadoop yarn tar ball +## @audience public +## @stability stable +function install_yarn_tarball_insecure() +{ + tag=`date '+%Y%m%d%H%M%S'` + if [ -f "${PACKAGE_DIR}/hadoop/${HADOOP_TARBALL}" ]; then + tar -zxvf "${PACKAGE_DIR}/hadoop/${HADOOP_TARBALL}" -C "${PACKAGE_DIR}/hadoop/" + mv "${PACKAGE_DIR}/hadoop/${HADOOP_VERSION}" "/home/${HADOOP_SETUP_USER}/${HADOOP_VERSION}-${tag}" + chown -R ${HADOOP_SETUP_USER} "/home/hadoop/${HADOOP_VERSION}-${tag}" + if [[ -d "${HADOOP_HOME}" ]] || [[ -L "${HADOOP_HOME}" ]]; then + rm -rf ${HADOOP_HOME} + fi + ln -s "/home/hadoop/${HADOOP_VERSION}-${tag}" "${HADOOP_HOME}" + chown ${HADOOP_SETUP_USER} "${HADOOP_HOME}" + else + echo "ERROR: Please put ${HADOOP_TARBALL} in the path of ${PACKAGE_DIR}/hadoop/ firstly." + return 1 + fi +} + +## @description Deploy java tar ball +## @audience public +## @stability stable +function install_java_tarball() +{ + if [[ -d "${JAVA_HOME}" ]] || [[ -L "${JAVA_HOME}" ]]; then + echo "JAVA_HOME already exists. There is no need to install java." + else + if [[ -f "${PACKAGE_DIR}/java/${JAVA_TARBALL}" ]]; then + tar -zxvf "${PACKAGE_DIR}/java/${JAVA_TARBALL}" -C "${PACKAGE_DIR}/java/" + mv "${PACKAGE_DIR}/java/${JAVA_VERSION}" "/home/${HADOOP_SETUP_USER}/${JAVA_VERSION}" + chown -R ${HADOOP_SETUP_USER} "/home/hadoop/${JAVA_VERSION}" + ln -s "/home/hadoop/${JAVA_VERSION}" "${JAVA_HOME}" + else + echo "Error: Failed to install java, please put java tarball in the path of + ${PACKAGE_DIR}/java/${JAVA_TARBALL}" + return 1 + fi + fi +} + +## @description install yarn resource & node manager +## @audience public +## @stability stable +function install_yarn_rm_nm_insecure() +{ + echo "install yarn config file ..." + host=$(hostname) + + find="/" + replace="\\/" + escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace} + escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace} + escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace} + escape_yarn_nodemanager_nodes_exclude_path=${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH//$find/$replace} + escape_yarn_nodemanager_recovery_dir=${YARN_NODEMANAGER_RECOVERY_DIR//$find/$replace} + escape_fs_defaults=${FS_DEFAULTFS//$find/$replace} + + # container-executor.cfg` + sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + + # enable cgroup for yarn + . "${PACKAGE_DIR}/submarine/submarine.sh" + + # Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error. + sed -i '1,16d' "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" + + executor_dir=$(dirname "${YARN_CONTAINER_EXECUTOR_PATH}") + executor_conf_dir=$(dirname "${executor_dir}")/etc/hadoop + if [ ! -d "${executor_conf_dir}" ]; then + sudo mkdir -p "${executor_conf_dir}" + fi + + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/container-executor.cfg" "${executor_conf_dir}" + + # yarn-site.xml + sed -i "s/YARN_RESOURCE_MANAGER_HOSTS1_REPLACE/${YARN_RESOURCE_MANAGER_HOSTS[0]}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_RESOURCE_MANAGER_HOSTS2_REPLACE/${YARN_RESOURCE_MANAGER_HOSTS[1]}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/LOCAL_CLUSTER_ID_REPLACE/${LOCAL_CLUSTER_ID}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + # Make nodemanager local dirs if the host is not in YARN_NODE_MANAGER_EXCLUDE_HOSTS + index=$(indexOfNMExcludeHosts "${host}") + if [ -z "$index" ]; then + arr=(${YARN_NODEMANAGER_LOCAL_DIRS//,/ }) + index=0 + while [ "$index" -lt "${#arr[@]}" ]; do + mkdir -p "${arr[$index]}" + (( index++ )) + done + + arr=(${YARN_NODEMANAGER_LOG_DIRS//,/ }) + index=0 + while [ "$index" -lt "${#arr[@]}" ]; do + mkdir -p "${arr[$index]}" + (( index++ )) + done + + arr=(${YARN_NODEMANAGER_LOCAL_HOME_PATHS//,/ }) + index=0 + while [ "$index" -lt "${#arr[@]}" ]; do + chown -R yarn "${arr[$index]}" + (( index++ )) + done + fi + + sed -i "s/YARN_ZK_ADDRESS_REPLACE/${YARN_ZK_ADDRESS}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + + sed -i "s/YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH_REPLACE/${escape_yarn_nodemanager_nodes_exclude_path}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + node_exclude_dir=$(dirname "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}") + if [[ ! -d "${node_exclude_dir}" ]]; then + mkdir -p "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" + fi + if [[ ! -f "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" ]]; then + touch "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" + chmod 777 "${YARN_RESOURCEMANAGER_NODES_EXCLUDE_PATH}" + fi + + sed -i "s/YARN_NODEMANAGER_RECOVERY_DIR_REPLACE/${escape_yarn_nodemanager_recovery_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + mkdir -p "${YARN_NODEMANAGER_RECOVERY_DIR}" + chmod 777 "${YARN_NODEMANAGER_RECOVERY_DIR}" + + # core-site.xml + sed -i "s/YARN_ZK_ADDRESS_REPLACE/${YARN_ZK_ADDRESS}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + sed -i "s/FS_DEFAULTFS_REPLACE/${escape_fs_defaults}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + + # WARN: ${HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE} Can not be empty! + echo 'hello submarine' > "${HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE}" + escape_hadoop_http_authentication_signature_secret_file=${HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE//$find/$replace} + sed -i "s/HADOOP_HTTP_AUTHENTICATION_SIGNATURE_SECRET_FILE_REPLACE/${escape_hadoop_http_authentication_signature_secret_file}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" + + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/core-site.xml" "${HADOOP_HOME}/etc/hadoop/" + cp -f "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/hdfs-site.xml" "${HADOOP_HOME}/etc/hadoop/" + + install_yarn_container_executor_insecure +} + +function install_spark_shuffle_insecure() { + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/spark* "${HADOOP_HOME}/share/hadoop/yarn/lib/" +} + +function install_lzo_native_insecure() { + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/native/libgpl* "${HADOOP_HOME}/lib/native/" + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/yarn/lib/" + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/hdfs/lib/" + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/common/lib/" + if [ ! -d "${HADOOP_HOME}/share/hadoop/mapreduce/lib/" ]; then + mkdir -p "${HADOOP_HOME}/share/hadoop/mapreduce/lib/" + fi + cp -R ${PACKAGE_DIR}/hadoop/yarn/lib/hadoop-lzo* "${HADOOP_HOME}/share/hadoop/mapreduce/lib/" +} + +function install_mapred_insecure() { + find="/" + replace="\\/" + escape_yarn_app_mapreduce_am_staging_dir=${YARN_APP_MAPREDUCE_AM_STAGING_DIR//$find/$replace} + escape_fs_defaults=${FS_DEFAULTFS//$find/$replace} + + sed -i "s/FS_DEFAULTFS_REPLACE/${escape_fs_defaults}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" + sed -i "s/YARN_APP_MAPREDUCE_AM_STAGING_DIR_REPLACE/${escape_yarn_app_mapreduce_am_staging_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" + host=$(hostname) + index=$(indexByRMHosts "${host}") + if [[ -n "$index" ]]; then + # Only RM needs to execute the following code + pathExists=$(pathExitsOnHDFS "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}") + if [[ -z "${pathExists}" ]]; then + echo "Create hdfs path ${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + "${HADOOP_HOME}/bin/hadoop" dfs -mkdir -p "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + "${HADOOP_HOME}/bin/hadoop" dfs -chown yarn:hadoop "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + "${HADOOP_HOME}/bin/hadoop" dfs -chmod 1777 "${YARN_APP_MAPREDUCE_AM_STAGING_DIR}" + fi + fi +} + +function install_yarn_sbin_insecure() { + find="/" + replace="\\/" + escape_yarn_gc_log_dir=${YARN_GC_LOG_DIR//$find/$replace} + escape_java_home=${JAVA_HOME//$find/$replace} + escape_hadoop_home=${HADOOP_HOME//$find/$replace} + escape_yarn_pid_dir=${YARN_PID_DIR//$find/$replace} + escape_yarn_log_dir=${YARN_LOG_DIR//$find/$replace} + cp -R ${PACKAGE_DIR}/hadoop/yarn/sbin/* "${HADOOP_HOME}/sbin/" + chown "${HADOOP_SETUP_USER}":yarn "${HADOOP_HOME}"/sbin/* + + if [ ! -d "$YARN_GC_LOG_DIR" ]; then + mkdir -p "$YARN_GC_LOG_DIR" + chown "${HADOOP_SETUP_USER}":yarn "${YARN_GC_LOG_DIR}" + chmod 775 "${YARN_GC_LOG_DIR}" + fi + + if [ ! -d "$YARN_LOG_DIR" ]; then + mkdir -p "$YARN_LOG_DIR" + chown "${HADOOP_SETUP_USER}":yarn "${YARN_LOG_DIR}" + chmod 775 "${YARN_LOG_DIR}" + fi + + if [ ! -d "$YARN_PID_DIR" ]; then + mkdir -p "$YARN_PID_DIR" + chown "${HADOOP_SETUP_USER}":yarn "${YARN_PID_DIR}" + chmod 775 "${YARN_PID_DIR}" + fi + + sed -i "s/YARN_LOG_DIR_REPLACE/${escape_yarn_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/YARN_PID_DIR_REPLACE/${escape_yarn_pid_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/JAVA_HOME_REPLACE/${escape_java_home}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/HADOOP_HOME_REPLACE/${escape_hadoop_home}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + sed -i "s/GC_LOG_DIR_REPLACE/${escape_yarn_gc_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" + cp -R "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/hadoop-env.sh" "${HADOOP_HOME}/etc/hadoop/" + + sed -i "s/GC_LOG_DIR_REPLACE/${escape_yarn_gc_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/yarn-env.sh" + cp -R "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/yarn-env.sh" "${HADOOP_HOME}/etc/hadoop/" + + sed -i "s/GC_LOG_DIR_REPLACE/${escape_yarn_gc_log_dir}/g" "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/mapred-env.sh" + cp -R "${INSTALL_TEMP_DIR}/hadoop/yarn/etc/hadoop/mapred-env.sh" "${HADOOP_HOME}/etc/hadoop/" + +cat< +hadoop.proxyuser.yarn.hosts +* + + +hadoop.proxyuser.yarn.groups +* + +----------------------------------------------------------------- +HELPINFO +} + +function install_registry_dns_insecure() { + sed -i "s/YARN_REGISTRY_DNS_HOST_REPLACE/${YARN_REGISTRY_DNS_HOST}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_REGISTRY_DNS_HOST_PORT_REPLACE/${YARN_REGISTRY_DNS_HOST_PORT}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" +} + +function install_job_history_insecure() { + sed -i "s/YARN_JOB_HISTORY_HOST_REPLACE/${YARN_JOB_HISTORY_HOST}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/mapred-site.xml" +} + +## @description install yarn timeline server +## @audience public +## @stability stable +## http://hadoop.apache.org/docs/r3.1.0/hadoop-yarn/hadoop-yarn-site/TimelineServer.html +function install_timeline_server_insecure() +{ + find="/" + replace="\\/" + escape_aggregated_log_dir=${YARN_AGGREGATED_LOG_DIR//$find/$replace} + escape_yarn_timeline_service_hbase_configuration_file=${YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE//$find/$replace} + escape_yarn_timeline_fs_store_dir=${YARN_TIMELINE_FS_STORE_DIR//$find/$replace} + # timeline v1.5 + escape_yarn_timeline_service_leveldb_state_store_path=${YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH//$find/$replace} + + # set leveldb configuration + sed -i "s/YARN_AGGREGATED_LOG_DIR_REPLACE/${escape_aggregated_log_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE_REPLACE/${escape_yarn_timeline_service_hbase_configuration_file}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + # timeline v1.5 + sed -i "s/YARN_TIMELINE_HOST_REPLACE/${YARN_TIMELINE_HOST}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH_REPLACE/${escape_yarn_timeline_service_leveldb_state_store_path}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + sed -i "s/YARN_TIMELINE_FS_STORE_DIR_REPLACE/${escape_yarn_timeline_fs_store_dir}/g" "$INSTALL_TEMP_DIR/hadoop/yarn/etc/hadoop/yarn-site.xml" + + host=$(hostname) + if [ "x$YARN_TIMELINE_HOST" != "x$host" ]; then + return 0 + fi + + echo -n "Do you want to create hdfs directories for timelineserver[Y/N]?" + read -r answer + echo "$answer" + if [[ "$answer" = "y" || "$answer" = "Y" ]]; then + echo "Continue installing ..." + else + echo "Stop creating hdfs directories for timelineserver" + return 0 + fi + + echo "install yarn timeline server V1.5 ..." + +cat< grant 'yarn', 'RWC' +> grant 'HTTP', 'R' +HELPINFO + echo -n "Have you done the above operation[Y/N]?" + read -r answer + if [[ "$answer" = "y" || "$answer" = "Y" ]]; then + echo "Continue installing ..." + else + echo "Stop installing the timeline server V2" + return 0 + fi + + pathExists=$(pathExitsOnHDFS "/hbase") + if [[ -z "${pathExists}" ]]; then + "${HADOOP_HOME}/bin/hadoop" fs -mkdir -p "/hbase" + "${HADOOP_HOME}/bin/hadoop" fs -chmod -R 755 "/hbase" + fi + + pathExists=$(pathExitsOnHDFS "/hbase/coprocessor") + if [[ -z "${pathExists}" ]]; then + "${HADOOP_HOME}/bin/hadoop" fs -mkdir -p "/hbase/coprocessor" + "${HADOOP_HOME}/bin/hadoop" fs -chmod -R 755 "/hbase/coprocessor" + fi + + "${HADOOP_HOME}/bin/hadoop" fs -put "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-hbase-coprocessor-3.*.jar "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar" + "${HADOOP_HOME}/bin/hadoop" fs -chmod 755 "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar" + +cat</lib path: +HELPINFO + + if [[ -n "${HBASE_HOME}" ]]; then + cp "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-hbase-common-3.*-SNAPSHOT.jar "${HBASE_HOME}/lib" + cp "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-hbase-client-3.*-SNAPSHOT.jar "${HBASE_HOME}/lib" + cp "${HADOOP_HOME}"/share/hadoop/yarn/timelineservice/hadoop-yarn-server-timelineservice-3.*-SNAPSHOT.jar "${HBASE_HOME}/lib" + fi + +cat< path, Execute the following command to create a schema +> bin/hbase org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator -create +HELPINFO + echo -n "Have you done the above operation[Y/N]?" + read -r answer + if [[ "$answer" = "y" || "$answer" = "Y" ]]; then + echo "Continue installing ..." + else + echo "Please initialize hbase timeline schema before you start timelineserver" + fi +} + +## @description start yarn +## @audience public +## @stability stable +function start_yarn() +{ + current_user=$(whoami) + host=$(hostname) + + # Start RM if the host is in YARN_RESOURCE_MANAGER_HOSTS + index=$(indexByRMHosts "${host}") + if [ -n "$index" ]; then + # Only RM needs to execute the following code + echo "Starting resourcemanager..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-resourcemanager.sh" + else + "${HADOOP_HOME}/sbin/start-resourcemanager.sh" + fi + fi + + # Start nodemanager + index=$(indexOfNMExcludeHosts "${host}") + if [ -z "$index" ]; then + echo "Starting nodemanager..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-nodemanager.sh" + else + "${HADOOP_HOME}/sbin/start-nodemanager.sh" + fi + fi + + # Start timeline + if [ "x$YARN_TIMELINE_HOST" = "x$host" ]; then + echo "Starting timelineserver..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-timelineserver.sh" + su - yarn -c "${HADOOP_HOME}/sbin/start-timelinereader.sh" + else + ${HADOOP_HOME}/sbin/start-timelineserver.sh + ${HADOOP_HOME}/sbin/start-timelinereader.sh + fi + fi + + # Start jobhistory + if [ "x$YARN_JOB_HISTORY_HOST" = "x$host" ]; then + echo "Starting mapreduce job history..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/start-mr-jobhistory.sh" + else + ${HADOOP_HOME}/sbin/start-mr-jobhistory.sh + fi + fi + + # Start registrydns + if [ "x$YARN_REGISTRY_DNS_HOST" = "x$host" ]; then + echo "Starting registry dns..." + sudo ${HADOOP_HOME}/sbin/start-registrydns.sh + fi +} + +## @description stop yarn +## @audience public +## @stability stable +function stop_yarn() +{ + current_user=$(whoami) + host=$(hostname) + # Stop RM if the host is in YARN_RESOURCE_MANAGER_HOSTS + index=$(indexByRMHosts "${host}") + if [ -n "$index" ]; then + # Only RM needs to execute the following code + if [ ${current_user} != "yarn" ]; then + echo "Stopping resourcemanager..." + su - yarn -c "${HADOOP_HOME}/sbin/stop-resourcemanager.sh" + else + "${HADOOP_HOME}/sbin/stop-resourcemanager.sh" + fi + fi + + # Stop nodemanager + index=$(indexOfNMExcludeHosts "${host}") + if [ -z "$index" ]; then + echo "Stopping nodemanager..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/stop-nodemanager.sh" + else + "${HADOOP_HOME}/sbin/stop-nodemanager.sh" + fi + fi + + # Stop timeline + if [ "x$YARN_TIMELINE_HOST" = "x$host" ]; then + echo "Stopping timelineserver..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/stop-timelineserver.sh" + su - yarn -c "${HADOOP_HOME}/sbin/stop-timelinereader.sh" + else + ${HADOOP_HOME}/sbin/stop-timelineserver.sh + ${HADOOP_HOME}/sbin/stop-timelinereader.sh + fi + fi + + # Stop jobhistory + if [ "x$YARN_JOB_HISTORY_HOST" = "x$host" ]; then + echo "Stopping mapreduce job history..." + if [ ${current_user} != "yarn" ]; then + su - yarn -c "${HADOOP_HOME}/sbin/stop-mr-jobhistory.sh" + else + ${HADOOP_HOME}/sbin/stop-mr-jobhistory.sh + fi + fi + + # Stop registrydns + if [ "x$YARN_REGISTRY_DNS_HOST" = "x$host" ]; then + echo "Stopping registry dns..." + sudo ${HADOOP_HOME}/sbin/stop-registrydns.sh + fi +} + diff --git a/dev-support/travis/install_external_dependencies.sh b/dev-support/travis/install_external_dependencies.sh new file mode 100755 index 0000000000000000000000000000000000000000..c73f28b780b92c587c537cc23e524ea207bdf368 --- /dev/null +++ b/dev-support/travis/install_external_dependencies.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Script for installing R / Python dependencies for Travis CI +set -exvuo pipefail +touch ~/.environ + +# Install Python dependencies for Python specific tests +if [[ -n "${PYTHON:-}" ]] ; then + wget https://repo.continuum.io/miniconda/Miniconda${PYTHON}-4.5.4-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "export PATH='$HOME/miniconda/bin:$PATH'" >> ~/.environ + source ~/.environ + + hash -r + conda config --set always_yes yes --set changeps1 no + conda update -q conda + conda info -a + conda config --add channels conda-forge + + conda install numpy=1.13.3 pandas=0.21.1 matplotlib=2.1.1 pandasql=0.7.3 ipython=5.4.1 jupyter_client=5.1.0 ipykernel=4.7.0 bokeh=0.12.10 + pip install scipy==1.5.2 ggplot==0.11.5 grpcio==1.8.2 bkzep==0.4.0 + + if [[ -n "${TENSORFLOW:-}" ]] ; then + check_results=`conda search -c conda-forge tensorflow` + echo "search tensorflow = $check_results" + + pip install tensorflow=="${TENSORFLOW}" + fi +fi diff --git a/dev-support/travis/tf-operator/crd_v1.yaml b/dev-support/travis/tf-operator/crd_v1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..546e66ac0b3e9648fb5ea79611075ac35ec8a470 --- /dev/null +++ b/dev-support/travis/tf-operator/crd_v1.yaml @@ -0,0 +1,42 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tfjobs.kubeflow.org +spec: + group: kubeflow.org + scope: Namespaced + names: + kind: TFJob + singular: tfjob + plural: tfjobs + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + tfReplicaSpecs: + properties: + # The validation works when the configuration contains + # `Worker`, `PS` or `Chief`. Otherwise it will not be validated. + Worker: + properties: + replicas: + type: integer + minimum: 1 + PS: + properties: + replicas: + type: integer + minimum: 1 + Chief: + properties: + replicas: + type: integer + minimum: 1 + maximum: 1 \ No newline at end of file diff --git a/dev-support/travis/tf-operator/tfevent-volume/tfevent-pv.yaml b/dev-support/travis/tf-operator/tfevent-volume/tfevent-pv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a450c6a49298bc8c8ff39eea0be71dc272e3f0c0 --- /dev/null +++ b/dev-support/travis/tf-operator/tfevent-volume/tfevent-pv.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: tfevent-volume + labels: + type: local + app: tfjob +spec: + capacity: + storage: 10Gi + storageClassName: standard + accessModes: + - ReadWriteMany + hostPath: + path: /tmp/data diff --git a/dev-support/travis/tf-operator/tfevent-volume/tfevent-pvc.yaml b/dev-support/travis/tf-operator/tfevent-volume/tfevent-pvc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81c2ca944fadfe10bdb0883fc6aedd86dff4ba7d --- /dev/null +++ b/dev-support/travis/tf-operator/tfevent-volume/tfevent-pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tfevent-volume + namespace: submarine + labels: + type: local + app: tfjob +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi diff --git a/docs/assets/128-black-white.png b/docs/assets/128-black-white.png new file mode 100644 index 0000000000000000000000000000000000000000..510a75e2c42c601021731ec6e43acc537c202bc6 Binary files /dev/null and b/docs/assets/128-black-white.png differ diff --git a/docs/assets/128-black.png b/docs/assets/128-black.png new file mode 100644 index 0000000000000000000000000000000000000000..f20d18565796c743ec8b86af6f4c894ad04b495f Binary files /dev/null and b/docs/assets/128-black.png differ diff --git a/docs/assets/128-white.png b/docs/assets/128-white.png new file mode 100644 index 0000000000000000000000000000000000000000..412d440c9b450277d38d0354c9943b7e5dfa4e68 Binary files /dev/null and b/docs/assets/128-white.png differ diff --git a/docs/assets/128.png b/docs/assets/128.png new file mode 100644 index 0000000000000000000000000000000000000000..3b1af38cd28be5d5f39cb9548b494d7065df37f6 Binary files /dev/null and b/docs/assets/128.png differ diff --git a/docs/assets/256-black-white.png b/docs/assets/256-black-white.png new file mode 100644 index 0000000000000000000000000000000000000000..315438bc14e272ab84e38055290df535d8517379 Binary files /dev/null and b/docs/assets/256-black-white.png differ diff --git a/docs/assets/256-black.png b/docs/assets/256-black.png new file mode 100644 index 0000000000000000000000000000000000000000..4813aea040986f7fb7e6b9a3e501e96e3f83764f Binary files /dev/null and b/docs/assets/256-black.png differ diff --git a/docs/assets/256-white-backup.png b/docs/assets/256-white-backup.png new file mode 100644 index 0000000000000000000000000000000000000000..1dad26e17c87251659a9cd334fff892b77661cb4 Binary files /dev/null and b/docs/assets/256-white-backup.png differ diff --git a/docs/assets/256-white.png b/docs/assets/256-white.png new file mode 100644 index 0000000000000000000000000000000000000000..7a10402a4a8ad2538a61db0f844135a994056506 Binary files /dev/null and b/docs/assets/256-white.png differ diff --git a/docs/assets/256.png b/docs/assets/256.png new file mode 100644 index 0000000000000000000000000000000000000000..ff55ec2c17bbb9de800d5fd2cb9944ed0df6b3d0 Binary files /dev/null and b/docs/assets/256.png differ diff --git a/docs/assets/32-black-white.png b/docs/assets/32-black-white.png new file mode 100644 index 0000000000000000000000000000000000000000..a15b54a768519071224385b0d922ae452c1da6a7 Binary files /dev/null and b/docs/assets/32-black-white.png differ diff --git a/docs/assets/32-black.png b/docs/assets/32-black.png new file mode 100644 index 0000000000000000000000000000000000000000..732d48423b0a4f61421fa30b078121a134c94588 Binary files /dev/null and b/docs/assets/32-black.png differ diff --git a/docs/assets/32-white.png b/docs/assets/32-white.png new file mode 100644 index 0000000000000000000000000000000000000000..ab0a2559cf46293e6479dc850641746d9e0b7f9e Binary files /dev/null and b/docs/assets/32-white.png differ diff --git a/docs/assets/32.png b/docs/assets/32.png new file mode 100644 index 0000000000000000000000000000000000000000..5acd270033df0976bf072bfebb091c86efc4a425 Binary files /dev/null and b/docs/assets/32.png differ diff --git a/docs/assets/64-black-white.png b/docs/assets/64-black-white.png new file mode 100644 index 0000000000000000000000000000000000000000..336c9192de201a6c663919c4ea88d3c1b9230689 Binary files /dev/null and b/docs/assets/64-black-white.png differ diff --git a/docs/assets/64-black.png b/docs/assets/64-black.png new file mode 100644 index 0000000000000000000000000000000000000000..e311f59e13164630b758e616e9af30c8da3abdb7 Binary files /dev/null and b/docs/assets/64-black.png differ diff --git a/docs/assets/64-white.png b/docs/assets/64-white.png new file mode 100644 index 0000000000000000000000000000000000000000..87790723cf5722b84084999aee665bdca254194b Binary files /dev/null and b/docs/assets/64-white.png differ diff --git a/docs/assets/64.png b/docs/assets/64.png new file mode 100644 index 0000000000000000000000000000000000000000..d53b6014a9c1bd8496d5efea585312a44cff6f14 Binary files /dev/null and b/docs/assets/64.png differ diff --git a/docs/assets/LOGO-black.svg b/docs/assets/LOGO-black.svg new file mode 100644 index 0000000000000000000000000000000000000000..fd443318ecc7f98b7268126e5d13dc8685ec1635 --- /dev/null +++ b/docs/assets/LOGO-black.svg @@ -0,0 +1,80 @@ + + + + +LOGO 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/LOGO-blue.svg b/docs/assets/LOGO-blue.svg new file mode 100644 index 0000000000000000000000000000000000000000..2558ffa0e18493161b42f2761c90359798eae341 --- /dev/null +++ b/docs/assets/LOGO-blue.svg @@ -0,0 +1,80 @@ + + + + +LOGO 3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/LOGO-white.svg b/docs/assets/LOGO-white.svg new file mode 100644 index 0000000000000000000000000000000000000000..76a614ca084970a83601d74ec90ab191a8f268b3 --- /dev/null +++ b/docs/assets/LOGO-white.svg @@ -0,0 +1,80 @@ + + + + +LOGO 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/Run-Debug-Configuration.png b/docs/assets/Run-Debug-Configuration.png new file mode 100644 index 0000000000000000000000000000000000000000..9bff9c255bc50dcfe9537c7c13ee2c952eab4a2e Binary files /dev/null and b/docs/assets/Run-Debug-Configuration.png differ diff --git a/docs/assets/apache_incubator.png b/docs/assets/apache_incubator.png new file mode 100644 index 0000000000000000000000000000000000000000..987c79e3ad3130904ea4ea5e3062d59d2bae805b Binary files /dev/null and b/docs/assets/apache_incubator.png differ diff --git a/docs/assets/architecture.png b/docs/assets/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..1949499e37109abd47269d19cd3d19b0686911c8 Binary files /dev/null and b/docs/assets/architecture.png differ diff --git a/docs/assets/asf_feather.png b/docs/assets/asf_feather.png new file mode 100644 index 0000000000000000000000000000000000000000..c0ebf11a6478c337f6897cd2ab8a221bd0fccd3f Binary files /dev/null and b/docs/assets/asf_feather.png differ diff --git a/docs/assets/color_logo_with_text.png b/docs/assets/color_logo_with_text.png new file mode 100644 index 0000000000000000000000000000000000000000..88b28f8a1bec05d2f764ab24a1666905ef71b3c3 Binary files /dev/null and b/docs/assets/color_logo_with_text.png differ diff --git a/docs/assets/color_logo_with_text.psd b/docs/assets/color_logo_with_text.psd new file mode 100644 index 0000000000000000000000000000000000000000..59d176a831f225c24c84c22ddcc7ee82e8e0ee15 Binary files /dev/null and b/docs/assets/color_logo_with_text.psd differ diff --git a/docs/assets/created-notebook.png b/docs/assets/created-notebook.png new file mode 100644 index 0000000000000000000000000000000000000000..6d08f3f61fdb43e3dde457a7e1a9b4dc70a43337 Binary files /dev/null and b/docs/assets/created-notebook.png differ diff --git a/docs/assets/design/experiments.png b/docs/assets/design/experiments.png new file mode 100644 index 0000000000000000000000000000000000000000..f3deb321f319c02f57b80160e509980a65c47874 Binary files /dev/null and b/docs/assets/design/experiments.png differ diff --git a/docs/assets/design/multi-dc-cloud.png b/docs/assets/design/multi-dc-cloud.png new file mode 100644 index 0000000000000000000000000000000000000000..6b5cfbb2eff1a82a3b6c88b2c8f6ec06679d5e88 Binary files /dev/null and b/docs/assets/design/multi-dc-cloud.png differ diff --git a/docs/assets/job-logs-ui.png b/docs/assets/job-logs-ui.png new file mode 100644 index 0000000000000000000000000000000000000000..18b9e17f10762f2d6e5ba2acaa5658d71efd2007 Binary files /dev/null and b/docs/assets/job-logs-ui.png differ diff --git a/docs/assets/logo-UEDC.psd b/docs/assets/logo-UEDC.psd new file mode 100644 index 0000000000000000000000000000000000000000..bbc938d6c54eb2f6e4c1ac8ea07c999768b934e2 Binary files /dev/null and b/docs/assets/logo-UEDC.psd differ diff --git a/docs/assets/logo-org.psd b/docs/assets/logo-org.psd new file mode 100644 index 0000000000000000000000000000000000000000..874edbab44fa43857665f3e2fe632311da1fecda Binary files /dev/null and b/docs/assets/logo-org.psd differ diff --git a/docs/assets/logo.psd b/docs/assets/logo.psd new file mode 100644 index 0000000000000000000000000000000000000000..9124ba0a46bf27e4e967adf348345c2f64621355 Binary files /dev/null and b/docs/assets/logo.psd differ diff --git a/docs/assets/logo.svg b/docs/assets/logo.svg new file mode 100644 index 0000000000000000000000000000000000000000..4bbdf7c8115fbc5264dc1e2acfa7b15972c22159 --- /dev/null +++ b/docs/assets/logo.svg @@ -0,0 +1 @@ +logo \ No newline at end of file diff --git a/docs/assets/multiple-tensorboard-jobs.png b/docs/assets/multiple-tensorboard-jobs.png new file mode 100644 index 0000000000000000000000000000000000000000..8e3db797269a56279a31a5fc5df185985301e713 Binary files /dev/null and b/docs/assets/multiple-tensorboard-jobs.png differ diff --git a/docs/assets/notebook-form.png b/docs/assets/notebook-form.png new file mode 100644 index 0000000000000000000000000000000000000000..04208ad7a46ea4bd10eaec0cf8200c8aa31fece9 Binary files /dev/null and b/docs/assets/notebook-form.png differ diff --git a/docs/assets/notebook-list.png b/docs/assets/notebook-list.png new file mode 100644 index 0000000000000000000000000000000000000000..6a90526d4e0898feccaf041c848f487e9933dc19 Binary files /dev/null and b/docs/assets/notebook-list.png differ diff --git a/docs/assets/submarine-installer.gif b/docs/assets/submarine-installer.gif new file mode 100644 index 0000000000000000000000000000000000000000..56b3b690f0ad4fd4f4d21c461bee6e1714070fbf Binary files /dev/null and b/docs/assets/submarine-installer.gif differ diff --git a/docs/assets/tensorboard-service.png b/docs/assets/tensorboard-service.png new file mode 100644 index 0000000000000000000000000000000000000000..3251d74531bf0668193cb8e7c48422c5a2e6d130 Binary files /dev/null and b/docs/assets/tensorboard-service.png differ diff --git a/docs/community/HowToCommit.md b/docs/community/HowToCommit.md new file mode 100644 index 0000000000000000000000000000000000000000..143e6bc52f730123b23aa3279611a208844131fc --- /dev/null +++ b/docs/community/HowToCommit.md @@ -0,0 +1,78 @@ + +# Guide for Apache Submarine Committers + +This page contains Hadoop Core-specific guidelines for committers. + +## New committers +New committers are encouraged to first read Apache's generic committer documentation: + +* [Apache New Committer Guide](http://www.apache.org/dev/new-committers-guide.html) +* [Apache Committer FAQ](http://www.apache.org/dev/committers.html) + +The first act of a new core committer is typically to add their name to the +credits page. This requires changing the site source in +https://github.com/apache/submarine-site/blob/master/community/member.md. Once done, +update the Submarine website as described +[here](https://github.com/apache/submarine-site/blob/asf-site/README.md) +(TLDR; don't forget to regenerate the site with hugo, and commit the generated +results, too). + +## Review +Submarine committers should, as often as possible, attempt to review patches +submitted by others. Ideally every submitted patch will get reviewed by a +committer within a few days. If a committer reviews a patch they've not +authored, and believe it to be of sufficient quality, then they can commit the +patch, otherwise the patch should be cancelled with a clear explanation for why +it was rejected. + +The list of submitted patches can be found in the GitHub +[Pull Requests](https://github.com/apache/submarine/pulls) page. +Committers should scan the list from top-to-bottom, +looking for patches that they feel qualified to review and possibly commit. + +For non-trivial changes, it is best to get another committer to review & approve +your own patches before commit. + +## Reject +Patches should be rejected which do not adhere to the guidelines in +[Contribution Guidelines](contributing.md). Committers should always be +polite to contributors and try to instruct and encourage them to contribute +better patches. If a committer wishes to improve an unacceptable patch, then it +should first be rejected, and a new patch should be attached by the committer +for review. + +## Commit individual patches +Submarine uses git for source code version control. The writable repo is at - +https://gitbox.apache.org/repos/asf/submarine.git + +It is strongly recommended to use the cicd script to merge the PRs. +See the instructions at +https://github.com/apache/submarine/tree/master/dev-support/cicd + +## Adding Contributors role +There are three roles (Administrators, Committers, Contributors) in the project. + +* Contributors who have Contributors role can become assignee of the issues in the project. +* Committers who have Committers role can set arbitrary roles in addition to Contributors role. +* Committers who have Administrators role can edit or delete all comments, or even delete issues in addition to Committers role. + +How to set roles + +1. Login to ASF JIRA +2. Go to the project page (e.g. https://issues.apache.org/jira/browse/SUBMARINE ) +3. Hit "Administration" tab +4. Hit "Roles" tab in left side +5. Add Administrators/Committers/Contributors role diff --git a/docs/community/README.md b/docs/community/README.md new file mode 100644 index 0000000000000000000000000000000000000000..160e9ee563beb8b166ceb837993ca32a4544865c --- /dev/null +++ b/docs/community/README.md @@ -0,0 +1,71 @@ + + +## Apache Submarine Community + +Welcome to the Apache Submarine Community! The main objective is to help members of the Submarine community who share similar interests to learn from and collaborate with each other. + +Your journey of becoming a contributor and committer starts from here: improving docs, improving code, giving talks, organizing meetups, etc. + +## Communicating + +You can reach out to the community members via any one of the following ways: + ++ Slack Developer: [https://the-asf.slack.com/submarine-dev/](https://the-asf.slack.com/submarine-dev/) + ++ Slack User: [https://the-asf.slack.com/submarine-user/](https://the-asf.slack.com/submarine-user/) + ++ Zoom: [https://cloudera.zoom.us/j/880548968](https://cloudera.zoom.us/j/880548968) + ++ Sync Up: [https://docs.google.com/document/d/16pUO3TP4SxSeLduG817GhVAjtiph9HYpRHo_JgduDvw/edit](https://docs.google.com/document/d/16pUO3TP4SxSeLduG817GhVAjtiph9HYpRHo_JgduDvw/edit) + +## Your First Contribution + +You can start by finding an existing issue with the [https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE?filter=allopenissues](https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE?filter=allopenissues) label. These issues are well suited for new contributors. + +If a PR (Pull Request) submitted to the [Submarine Github](https://github.com/apache/submarine) projects by you is approved and merged, then you become a Submarine Contributor. + +If you want to work on a new idea of relatively small scope: + +1. Submit an issue describing your proposed change to the repo in question. + +2. The repo owners will respond to your issue promptly. + +3. Submit a [pull request of Submarine](https://github.com/apache/submarine) containing a tested change. + +Once you become a contributor of Apache Submarine, check your name here: [CONTRIBUTORS](contributors.md) + +Contributions are welcomed and greatly appreciated. See [CONTRIBUTING.md](contributing.md) for details on submitting patches and the contribution workflow. + +## How Do I Become a Committer? + +First of all, you need to get involved and be a Contributor. + +Based on your track-record as a contributor, Per Apache code, PMCs vote on committership, may invite you to be a committer (after we've called a vote). When that happens, if you accept, the following process kicks into place... + +Note that becoming a committer is not just about submitting some patches; it‘s also about helping out on the development and user [Slack User](https://the-asf.slack.com/submarine-user/), helping with documentation and the issues. + +See [Become a Committer.md](become-a-committer.md) for steps of becoming a committer and more details. + +## How to commit + +See [How to commit](HowToCommit.md) for helper doc for Submarine committers. + +## Communication + +Communication within the Submarine community abides by [Apache’s Code of Conduct](https://www.apache.org/foundation/policies/conduct.html). + +## License + +Submarine source code is under the Apache 2.0 license. See the [LICENSE](https://github.com/apache/submarine/blob/master/LICENSE) file for details. diff --git a/docs/community/become-a-committer.md b/docs/community/become-a-committer.md new file mode 100644 index 0000000000000000000000000000000000000000..fb61776c391ce04314722660c444b9dd4f6281c5 --- /dev/null +++ b/docs/community/become-a-committer.md @@ -0,0 +1,15 @@ + + +# Become a Committer diff --git a/docs/community/contributing.md b/docs/community/contributing.md new file mode 100644 index 0000000000000000000000000000000000000000..e0fbfa9464375d17313b1848d21628c2e77e15ab --- /dev/null +++ b/docs/community/contributing.md @@ -0,0 +1,264 @@ + + +# Contribution Guidelines + +**Apache Hadoop Submarine** is an [Apache 2.0 License](https://github.com/apache/submarine/blob/master/LICENSE) Software. + +Contributing to Hadoop Submarine (Source code, Documents, Image, Website) means you agree to the Apache 2.0 License. + +1. Make sure your issue is not already in the [Jira issue tracker](https://issues.apache.org/jira/browse/SUBMARINE) +2. If not, create a ticket describing the change you're proposing in the [Jira issue tracker](https://issues.apache.org/jira/browse/SUBMARINE) +3. Setup Travis [Continuous Integration](#continuous-integration) +4. Contribute your patch via Pull Request on our [Github Mirror](https://github.com/apache/submarine). + +Before you start, please read the [Code of Conduct](http://www.apache.org/foundation/policies/conduct.html) carefully, familiarize yourself with it and refer to it whenever you need it. + +For those of you who are not familiar with the Apache project, understanding [How it works](http://www.apache.org/foundation/how-it-works.html) would be quite helpful. + +## Creating a Pull Request +When creating a Pull Request, you will automatically get the template below. + +Filling it thoroughly can improve the speed of the review process. + + ### What is this PR for? + A few sentences describing the overall goals of the pull request's commits. + First time? Check out the contribution guidelines - + https://github.com/apache/submarine/tree/master/docs/community/contributing.md + + ### What type of PR is it? + [Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring] + + ### Todos + * [ ] - Task + + ### What is the Jira issue? + * Open an issue on Jira https://issues.apache.org/jira/browse/SUBMARINE/ + * Put link here, and add [SUBMARINE-${jira_number}] in PR title, e.g. [SUBMARINE-323] + + ### How should this be tested? + Outline the steps to test the PR here. + + ### Screenshots (if appropriate) + + ### Questions: + * Do the licenses files require updates? + * Are there breaking changes for older versions? + * Does this need documentation? + + +## Source Control Workflow +Hadoop Submarine follows [Fork & Pull](https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) model. + +## The Review Process + +When a Pull Request is submitted, it is being merged or rejected by the following review process. + +* Anybody can be a reviewer and may comment on the change or suggest modifications. +* Reviewer can indicate that a patch looks suitable for merging with a comment such as: "Looks good", "LGTM", "+1". +* At least one indication of suitability (e.g. "LGTM") from a committer is required to be merged. +* Pull request is open for 1 or 2 days for potential additional review unless it's got enough indication of suitability. +* A committer can then initiate lazy consensus ("Merge if there is no more discussion") after which the code can be merged after a particular time (usually 24 hours) if there are no more reviews. +* Contributors can ping reviewers (including committers) by commenting 'Ready to review' or suitable indication. + + +## Setting up +Here are some things you will need to build and test the Hadoop Submarine. + +### Software Configuration Management (SCM) + +Hadoop Submarine uses Git for its SCM system. So you'll need a git client installed on your development machine. + +### Integrated Development Environment (IDE) + +You are free to use whatever IDE you prefer, or your favorite command-line editor. + +### Code convention +We are following Google Code style: + +* [Java style](https://google.github.io/styleguide/javaguide.html) +* [Shell style](https://google.github.io/styleguide/shell.xml) + +There are some plugins to format, lint your code in IDE (use [dev-support/maven-config/checkstyle.xml](hhttps://github.com/apache/submarine/blob/master/dev-support/maven-config/checkstyle.xml) as rules) + +* [Checkstyle plugin for Intellij](https://plugins.jetbrains.com/plugin/1065) ([Setting Guide](http://stackoverflow.com/questions/26955766/intellij-idea-checkstyle)) +* [Checkstyle plugin for Eclipse](http://eclipse-cs.sourceforge.net/#!/) ([Setting Guide](http://eclipse-cs.sourceforge.net/#!/project-setup)) + + +## Getting the source code + +### Step 1: Fork in the cloud + +1. Visit https://github.com/apache/submarine +2. On the top right of the page, click the `Fork` button (top right) to create a cloud-based fork of the repository. + +### Step 2: Clone fork to local storage + +Create your clone: + +> ${user} is your github user name + +```sh +mkdir -p ${working_dir} +cd ${working_dir} + +git clone https://github.com/${user}/submarine.git +# or: git clone git@github.com:${user}/submarine.git + +cd ${working_dir}/submarine +git remote add upstream https://github.com/apache/submarine.git +# or: git remote add upstream git@github.com:apache/submarine.git + +# Never push to the upstream master. +git remote set-url --push upstream no_push + +# Confirm that your remotes make sense: +# It should look like: +# origin git@github.com:${user}/submarine.git (fetch) +# origin git@github.com:${user}/submarine.git (push) +# upstream https://github.com/apache/submarine (fetch) +# upstream no_push (push) +git remote -v +``` + +### Step 3: Branch + +Get your local master up to date: + +```sh +cd ${working_dir}/submarine +git fetch upstream +git checkout master +git rebase upstream/master +``` + +Branch from master: + +```sh +git checkout -b SUBMARINE-${jira_number} +``` + +### Step 4: Develop + +#### Edit the code + +You can now edit the code on the `SUBMARINE-${jira_number}` branch. + +#### Test + +Build and run all tests: + +### Step 5: Keep your branch in sync + +```sh +# While on your SUBMARINE-${jira_number} branch. +git fetch upstream +git rebase upstream/master +``` + +Please don't use `git pull` instead of the above `fetch`/`rebase`. `git pull` does a merge, which leaves merge commits. These make the commit history messy and violate the principle that commits ought to be individually understandable and useful (see below). You can also consider changing your `.git/config` file via `git config branch.autoSetupRebase` always to change the behavior of `git pull`. + +### Step 6: Commit + +Commit your changes. + +```sh +git commit +``` + +Likely you'll go back and edit/build/test further and then `commit --amend` in a few cycles. + +### Step 7: Push + +When the changes are ready to review (or you just want to create an offsite backup of your work), push your branch to your fork on `github.com`: + +```sh +git push --set-upstream ${your_remote_name} SUBMARINE-${jira_number} +``` + +### Step 8: Create a pull request + +1. Visit your fork at `https://github.com/${user}/submarine`. +2. Click the `Compare & Pull Request` button next to your `SUBMARINE-${jira_number}` branch. +3. Fill in the required information in the PR template. + +#### Get a code review + +If your pull request (PR) is opened, it will be assigned to one or more reviewers. Those reviewers will do a thorough code review, looking at correctness, bugs, opportunities for improvement, documentation comments, and style. + +To address review comments, you should commit the changes to the same branch of the PR on your fork. + +#### Revert a commit + +In case you wish to revert a commit, follow the instructions below: + +> NOTE: If you have upstream write access, please refrain from using the Revert +> button in the GitHub UI for creating the PR, because GitHub will create the +> PR branch inside the main repository rather than inside your fork. + +Create a branch and synchronize it with the upstream: + +```sh +# create a branch +git checkout -b myrevert + +# sync the branch with upstream +git fetch upstream +git rebase upstream/master + +# SHA is the hash of the commit you wish to revert +git revert SHA +``` + +This creates a new commit reverting the change. Push this new commit to your remote: + +```sh +git push ${your_remote_name} myrevert +``` + +Create a PR based on this branch. + +#### Cherry pick a commit to a release branch + +In case you wish to cherry pick a commit to a release branch, follow the instructions below: + +Create a branch and synchronize it with the upstream: + +```sh +# sync the branch with upstream. +git fetch upstream + +# checkout the release branch. +# ${release_branch_name} is the release branch you wish to cherry pick to. +git checkout upstream/${release_branch_name} +git checkout -b my-cherry-pick + +# cherry pick the commit to my-cherry-pick branch. +# ${SHA} is the hash of the commit you wish to revert. +git cherry-pick ${SHA} + +# push this branch to your repo, file an PR based on this branch. +git push --set-upstream ${your_remote_name} my-cherry-pick +``` + +## Build + +[Build From Code](../development/BuildFromCode.md) + +## Continuous Integration + +Hadoop Submarine project's CI system will collect information from pull request author's Travis-ci and display status in the pull request. + +Each individual contributor should setup Travis-ci for the fork before making a pull-request. Go to [https://travis-ci.org/profile](https://travis-ci.org/profile) and switch on 'submarine' repository. + diff --git a/docs/community/contributors.md b/docs/community/contributors.md new file mode 100644 index 0000000000000000000000000000000000000000..8cf332e66c08955d5aee6bcdf8c5bbd39c3a9e39 --- /dev/null +++ b/docs/community/contributors.md @@ -0,0 +1,15 @@ + + +# Hadoop Submarine contributors diff --git a/docs/database/README.md b/docs/database/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bf0e4ad297d1899f2404eb0353e8df127722d119 --- /dev/null +++ b/docs/database/README.md @@ -0,0 +1,163 @@ + + +# Submarine Database + +Submarine needs to use the database to store information about the `organization`, `user`, `projects`, `tasks`, `metastore` and `configuration` of the system information, So consider using mysql to store this data. + ++ MySQL will be included in the `mini-submarine` docker image to allow users to quickly experience the `submarine workbench`. ++ In a production environment, the `submarine workbench` can be connected to the official mysql database. + +## Prerequisite + +Must: + +- MySQL +- MyBatis + +## Run mysql on docker + +By using the official docker image of submarine database, only one docker command is required to run submarine database + +```bash +docker run -it -p 3306:3306 -d --name submarine-database -e MYSQL_ROOT_PASSWORD=password apache/submarine:database- +``` +## Initialize submarine database +It will create users and tables that submarine requires +```shell script +sudo ./init-database +``` +## Manual operation of the submarine database + +### Modify character set (Optional) + +If you need to store Chinese character data in mysql, you need to execute the following command to modify the mysql character set. + ++ Set database character set + + ``` + bash > mysql -uroot -ppassword + + mysql>SHOW VARIABLES LIKE 'character_set_%'; // View database character set + mysql>SHOW VARIABLES LIKE 'collation_%'; + + SET NAMES 'utf8'; + ``` + ++ Configuration `mysqld.cnf` + + ``` + # install vim + apt-get update + apt-get install vim + + vi /etc/mysql/mysql.conf.d/mysqld.cnf + + [mysqld] + character_set_server = utf8 + + [mysql] + default-character-set = utf8 + + [mysql.server] + default-character-set = utf8 + + [mysqld_safe] + default-character-set = utf8 + + [client] + default-character-set = utf8 + ``` + +### Create Submarine Database + + +#### Create development database + +Run [mysql docker container](https://hub.docker.com/_/mysql) + +``` +docker run --name some-mysql -e MYSQL_ROOT_PASSWORD=my-secret-pw -d mysql:tag +``` + +Copy the files, submarine.sql, submarine-data.sql and metastore.sql to the mysql docker. + +``` +docker cp ${SUBMARINE_HOME}/docs/database/submarine.sql ${DOCKER_ID}:/ +docker cp ${SUBMARINE_HOME}/docs/database/submarine-data.sql ${DOCKER_ID}:/ +docker cp ${SUBMARINE_HOME}/docs/database/metastore.sql ${DOCKER_ID}:/ +``` + +Development database for development environment. + +``` +# in mysql container +bash > mysql -uroot -ppassword +mysql> CREATE USER IF NOT EXISTS 'submarine'@'%' IDENTIFIED BY 'password'; +mysql> GRANT ALL PRIVILEGES ON * . * TO 'submarine'@'%'; +mysql> CREATE DATABASE IF NOT EXISTS submarine CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> use submarine; +mysql> source /submarine.sql; +mysql> source /submarine-data.sql; +mysql> CREATE USER IF NOT EXISTS 'metastore'@'%' IDENTIFIED BY 'password'; +mysql> GRANT ALL PRIVILEGES ON * . * TO 'metastore'@'%'; +mysql> CREATE DATABASE IF NOT EXISTS metastore CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> use metastore; +mysql> source /metastore.sql; +mysql> quit +``` + +> NOTE: submarine development database name is `submarine` and user name is `submarine`, password is `password`, metastore development database name is `metastore` and user name is `metastore`, password is `password`, This is the default value in the system's `submarine-site.xml` configuration file and is not recommended for modification. + + +#### Create test database + +Test database for program unit testing and Travis test environment. + +``` +# in mysql container +bash > mysql -uroot -ppassword +mysql> CREATE USER IF NOT EXISTS 'submarine_test'@'%' IDENTIFIED BY 'password_test'; +mysql> GRANT ALL PRIVILEGES ON * . * TO 'submarine_test'@'%'; +mysql> CREATE DATABASE IF NOT EXISTS `submarine_test` CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> use `submarine_test`; +mysql> source /submarine.sql; +mysql> CREATE USER IF NOT EXISTS 'metastore_test'@'%' IDENTIFIED BY 'password_test'; +mysql> GRANT ALL PRIVILEGES ON * . * TO 'metastore_test'@'%'; +mysql> CREATE DATABASE IF NOT EXISTS `metastore_test` CHARACTER SET utf8 COLLATE utf8_general_ci; +mysql> use `metastore_test`; +mysql> source /metastore.sql; +mysql> quit +``` + +> NOTE: submarine test database name is `submarine_test` and user name is `submarine_test`, password is `password_test`, metastore test database name is `metastore_test` and user name is `metastore_test`, password is `password_test`, Cannot be configured, values that cannot be modified. + +#### mysqldump + +```$xslt +mysqldump -uroot -ppassword --databases submarine > submarine.sql; +mysqldump -umetastore -ppassword metastore > metastore.sql; +``` + + +## Travis + +1. In the submarine's Travis, the `test database`, `database name`, `username` and `password` will be automatically created based on the contents of this document. + + Therefore, do not modify the database's `database name`, `username` and `password` configuration to avoid introducing some problems. + +2. In the mysql database in Travis, the `submarine.sql` are executed to create the submarine database table structure and test data. + +3. The submarine database test case written in the `workbench-server` module will also be unit tested in the mysql database in travis. diff --git a/docs/database/init-database.sh b/docs/database/init-database.sh new file mode 100755 index 0000000000000000000000000000000000000000..f9bf04798e833da903852b83761bc469c5150751 --- /dev/null +++ b/docs/database/init-database.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FWDIR="$(cd "$(dirname "$0")"; pwd)" +cd "$FWDIR" + +mysql -e "CREATE DATABASE IF NOT EXISTS submarine_test;" +mysql -e "CREATE USER IF NOT EXISTS 'submarine_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';" +mysql -e "use submarine_test; source ./submarine.sql; show tables;" + +mysql -e "CREATE DATABASE IF NOT EXISTS metastore_test;" +mysql -e "CREATE USER IF NOT EXISTS 'metastore_test'@'%' IDENTIFIED BY 'password_test';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore_test'@'%';" +mysql -e "use metastore_test; source ./metastore.sql; show tables;" + +mysql -e "CREATE DATABASE IF NOT EXISTS submarine;" +mysql -e "CREATE USER IF NOT EXISTS 'submarine'@'%' IDENTIFIED BY 'password';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';" +mysql -e "use submarine; source ./submarine.sql; source ./submarine-data.sql; show tables;" + +mysql -e "CREATE DATABASE IF NOT EXISTS metastore;" +mysql -e "CREATE USER IF NOT EXISTS 'metastore'@'%' IDENTIFIED BY 'password';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore'@'%';" +mysql -e "use metastore; source ./metastore.sql; show tables;" diff --git a/docs/database/metastore.sql b/docs/database/metastore.sql new file mode 100644 index 0000000000000000000000000000000000000000..041c96d9ef265f767b1e557af1e09f142dd4b80e --- /dev/null +++ b/docs/database/metastore.sql @@ -0,0 +1,982 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- http://www.apache.org/licenses/LICENSE-2.0 +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- fork by https://github.com/apache/hive/blob/master/metastore/scripts/ +-- upgrade/mysql/hive-schema-2.3.0.mysql.sql + +-- fork by https://github.com/apache/hive/blob/master/metastore/scripts/ +-- upgrade/mysql/hive-txn-schema-2.3.0.mysql.sql + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `BUCKETING_COLS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `BUCKETING_COLS` ( + `SD_ID` bigint(20) NOT NULL, + `BUCKET_COL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), + KEY `BUCKETING_COLS_N49` (`SD_ID`), + CONSTRAINT `BUCKETING_COLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `CDS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `CDS` ( + `CD_ID` bigint(20) NOT NULL, + PRIMARY KEY (`CD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `COLUMNS_V2` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `COLUMNS_V2` ( + `CD_ID` bigint(20) NOT NULL, + `COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TYPE_NAME` MEDIUMTEXT DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`CD_ID`,`COLUMN_NAME`), + KEY `COLUMNS_V2_N49` (`CD_ID`), + CONSTRAINT `COLUMNS_V2_FK1` FOREIGN KEY (`CD_ID`) REFERENCES `CDS` (`CD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `DATABASE_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `DATABASE_PARAMS` ( + `DB_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(180) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`DB_ID`,`PARAM_KEY`), + KEY `DATABASE_PARAMS_N49` (`DB_ID`), + CONSTRAINT `DATABASE_PARAMS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `DBS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `DBS` ( + `DB_ID` bigint(20) NOT NULL, + `DESC` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `DB_LOCATION_URI` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `OWNER_TYPE` varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`DB_ID`), + UNIQUE KEY `UNIQUE_DATABASE` (`NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `DB_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `DB_PRIVS` ( + `DB_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `DB_ID` bigint(20) DEFAULT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `DB_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`DB_GRANT_ID`), + UNIQUE KEY `DBPRIVILEGEINDEX` (`DB_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`DB_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `DB_PRIVS_N49` (`DB_ID`), + CONSTRAINT `DB_PRIVS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `GLOBAL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( + `USER_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `USER_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`USER_GRANT_ID`), + UNIQUE KEY `GLOBALPRIVILEGEINDEX` (`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`USER_PRIV`,`GRANTOR`,`GRANTOR_TYPE`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `IDXS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `IDXS` ( + `INDEX_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `DEFERRED_REBUILD` bit(1) NOT NULL, + `INDEX_HANDLER_CLASS` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INDEX_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INDEX_TBL_ID` bigint(20) DEFAULT NULL, + `LAST_ACCESS_TIME` int(11) NOT NULL, + `ORIG_TBL_ID` bigint(20) DEFAULT NULL, + `SD_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`INDEX_ID`), + UNIQUE KEY `UNIQUEINDEX` (`INDEX_NAME`,`ORIG_TBL_ID`), + KEY `IDXS_N51` (`SD_ID`), + KEY `IDXS_N50` (`INDEX_TBL_ID`), + KEY `IDXS_N49` (`ORIG_TBL_ID`), + CONSTRAINT `IDXS_FK1` FOREIGN KEY (`ORIG_TBL_ID`) REFERENCES `TBLS` (`TBL_ID`), + CONSTRAINT `IDXS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), + CONSTRAINT `IDXS_FK3` FOREIGN KEY (`INDEX_TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `INDEX_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( + `INDEX_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`), + KEY `INDEX_PARAMS_N49` (`INDEX_ID`), + CONSTRAINT `INDEX_PARAMS_FK1` FOREIGN KEY (`INDEX_ID`) REFERENCES `IDXS` (`INDEX_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `NUCLEUS_TABLES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `NUCLEUS_TABLES` ( + `CLASS_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TYPE` varchar(4) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `OWNER` varchar(2) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `VERSION` varchar(20) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `INTERFACE_NAME` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`CLASS_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITIONS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITIONS` ( + `PART_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `LAST_ACCESS_TIME` int(11) NOT NULL, + `PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `SD_ID` bigint(20) DEFAULT NULL, + `TBL_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`PART_ID`), + UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), + KEY `PARTITIONS_N49` (`TBL_ID`), + KEY `PARTITIONS_N50` (`SD_ID`), + CONSTRAINT `PARTITIONS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`), + CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_EVENTS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_EVENTS` ( + `PART_NAME_ID` bigint(20) NOT NULL, + `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `EVENT_TIME` bigint(20) NOT NULL, + `EVENT_TYPE` int(11) NOT NULL, + `PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_NAME_ID`), + KEY `PARTITIONEVENTINDEX` (`PARTITION_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_KEYS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_KEYS` ( + `TBL_ID` bigint(20) NOT NULL, + `PKEY_COMMENT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PKEY_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PKEY_TYPE` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`TBL_ID`,`PKEY_NAME`), + KEY `PARTITION_KEYS_N49` (`TBL_ID`), + CONSTRAINT `PARTITION_KEYS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_KEY_VALS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( + `PART_ID` bigint(20) NOT NULL, + `PART_KEY_VAL` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`PART_ID`,`INTEGER_IDX`), + KEY `PARTITION_KEY_VALS_N49` (`PART_ID`), + CONSTRAINT `PARTITION_KEY_VALS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PARTITION_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( + `PART_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_ID`,`PARAM_KEY`), + KEY `PARTITION_PARAMS_N49` (`PART_ID`), + CONSTRAINT `PARTITION_PARAMS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PART_COL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` ( + `PART_COLUMN_GRANT_ID` bigint(20) NOT NULL, + `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_ID` bigint(20) DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_COL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_COLUMN_GRANT_ID`), + KEY `PART_COL_PRIVS_N49` (`PART_ID`), + KEY `PARTITIONCOLUMNPRIVILEGEINDEX` (`PART_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + CONSTRAINT `PART_COL_PRIVS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `PART_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `PART_PRIVS` ( + `PART_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_ID` bigint(20) DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PART_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`PART_GRANT_ID`), + KEY `PARTPRIVILEGEINDEX` (`PART_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `PART_PRIVS_N49` (`PART_ID`), + CONSTRAINT `PART_PRIVS_FK1` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `ROLES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `ROLES` ( + `ROLE_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `OWNER_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `ROLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`ROLE_ID`), + UNIQUE KEY `ROLEENTITYINDEX` (`ROLE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `ROLE_MAP` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `ROLE_MAP` ( + `ROLE_GRANT_ID` bigint(20) NOT NULL, + `ADD_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `ROLE_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`ROLE_GRANT_ID`), + UNIQUE KEY `USERROLEMAPINDEX` (`PRINCIPAL_NAME`,`ROLE_ID`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `ROLE_MAP_N49` (`ROLE_ID`), + CONSTRAINT `ROLE_MAP_FK1` FOREIGN KEY (`ROLE_ID`) REFERENCES `ROLES` (`ROLE_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SDS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SDS` ( + `SD_ID` bigint(20) NOT NULL, + `CD_ID` bigint(20) DEFAULT NULL, + `INPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `IS_COMPRESSED` bit(1) NOT NULL, + `IS_STOREDASSUBDIRECTORIES` bit(1) NOT NULL, + `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `NUM_BUCKETS` int(11) NOT NULL, + `OUTPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `SERDE_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`SD_ID`), + KEY `SDS_N49` (`SERDE_ID`), + KEY `SDS_N50` (`CD_ID`), + CONSTRAINT `SDS_FK1` FOREIGN KEY (`SERDE_ID`) REFERENCES `SERDES` (`SERDE_ID`), + CONSTRAINT `SDS_FK2` FOREIGN KEY (`CD_ID`) REFERENCES `CDS` (`CD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SD_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( + `SD_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` MEDIUMTEXT CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SD_ID`,`PARAM_KEY`), + KEY `SD_PARAMS_N49` (`SD_ID`), + CONSTRAINT `SD_PARAMS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SEQUENCE_TABLE` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( + `SEQUENCE_NAME` varchar(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `NEXT_VAL` bigint(20) NOT NULL, + PRIMARY KEY (`SEQUENCE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SERDES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SERDES` ( + `SERDE_ID` bigint(20) NOT NULL, + `NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `SLIB` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SERDE_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SERDE_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( + `SERDE_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` MEDIUMTEXT CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SERDE_ID`,`PARAM_KEY`), + KEY `SERDE_PARAMS_N49` (`SERDE_ID`), + CONSTRAINT `SERDE_PARAMS_FK1` FOREIGN KEY (`SERDE_ID`) REFERENCES `SERDES` (`SERDE_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_COL_NAMES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( + `SD_ID` bigint(20) NOT NULL, + `SKEWED_COL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), + KEY `SKEWED_COL_NAMES_N49` (`SD_ID`), + CONSTRAINT `SKEWED_COL_NAMES_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_COL_VALUE_LOC_MAP` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( + `SD_ID` bigint(20) NOT NULL, + `STRING_LIST_ID_KID` bigint(20) NOT NULL, + `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`SD_ID`,`STRING_LIST_ID_KID`), + KEY `SKEWED_COL_VALUE_LOC_MAP_N49` (`STRING_LIST_ID_KID`), + KEY `SKEWED_COL_VALUE_LOC_MAP_N50` (`SD_ID`), + CONSTRAINT `SKEWED_COL_VALUE_LOC_MAP_FK2` FOREIGN KEY (`STRING_LIST_ID_KID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`), + CONSTRAINT `SKEWED_COL_VALUE_LOC_MAP_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_STRING_LIST` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( + `STRING_LIST_ID` bigint(20) NOT NULL, + PRIMARY KEY (`STRING_LIST_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_STRING_LIST_VALUES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( + `STRING_LIST_ID` bigint(20) NOT NULL, + `STRING_LIST_VALUE` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`STRING_LIST_ID`,`INTEGER_IDX`), + KEY `SKEWED_STRING_LIST_VALUES_N49` (`STRING_LIST_ID`), + CONSTRAINT `SKEWED_STRING_LIST_VALUES_FK1` FOREIGN KEY (`STRING_LIST_ID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SKEWED_VALUES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` ( + `SD_ID_OID` bigint(20) NOT NULL, + `STRING_LIST_ID_EID` bigint(20) NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID_OID`,`INTEGER_IDX`), + KEY `SKEWED_VALUES_N50` (`SD_ID_OID`), + KEY `SKEWED_VALUES_N49` (`STRING_LIST_ID_EID`), + CONSTRAINT `SKEWED_VALUES_FK2` FOREIGN KEY (`STRING_LIST_ID_EID`) REFERENCES `SKEWED_STRING_LIST` (`STRING_LIST_ID`), + CONSTRAINT `SKEWED_VALUES_FK1` FOREIGN KEY (`SD_ID_OID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `SORT_COLS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `SORT_COLS` ( + `SD_ID` bigint(20) NOT NULL, + `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `ORDER` int(11) NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`SD_ID`,`INTEGER_IDX`), + KEY `SORT_COLS_N49` (`SD_ID`), + CONSTRAINT `SORT_COLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TABLE_PARAMS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( + `TBL_ID` bigint(20) NOT NULL, + `PARAM_KEY` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARAM_VALUE` MEDIUMTEXT CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`TBL_ID`,`PARAM_KEY`), + KEY `TABLE_PARAMS_N49` (`TBL_ID`), + CONSTRAINT `TABLE_PARAMS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TBLS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TBLS` ( + `TBL_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `DB_ID` bigint(20) DEFAULT NULL, + `LAST_ACCESS_TIME` int(11) NOT NULL, + `OWNER` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `RETENTION` int(11) NOT NULL, + `SD_ID` bigint(20) DEFAULT NULL, + `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `VIEW_EXPANDED_TEXT` mediumtext, + `VIEW_ORIGINAL_TEXT` mediumtext, + `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`TBL_ID`), + UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), + KEY `TBLS_N50` (`SD_ID`), + KEY `TBLS_N49` (`DB_ID`), + CONSTRAINT `TBLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`), + CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TBL_COL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( + `TBL_COLUMN_GRANT_ID` bigint(20) NOT NULL, + `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_COL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`TBL_COLUMN_GRANT_ID`), + KEY `TABLECOLUMNPRIVILEGEINDEX` (`TBL_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + KEY `TBL_COL_PRIVS_N49` (`TBL_ID`), + CONSTRAINT `TBL_COL_PRIVS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TBL_PRIVS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TBL_PRIVS` ( + `TBL_GRANT_ID` bigint(20) NOT NULL, + `CREATE_TIME` int(11) NOT NULL, + `GRANT_OPTION` smallint(6) NOT NULL, + `GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `GRANTOR_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `PRINCIPAL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_PRIV` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TBL_ID` bigint(20) DEFAULT NULL, + PRIMARY KEY (`TBL_GRANT_ID`), + KEY `TBL_PRIVS_N49` (`TBL_ID`), + KEY `TABLEPRIVILEGEINDEX` (`TBL_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`), + CONSTRAINT `TBL_PRIVS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TAB_COL_STATS` +-- +CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( + `CS_ID` bigint(20) NOT NULL, + `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TABLE_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TBL_ID` bigint(20) NOT NULL, + `LONG_LOW_VALUE` bigint(20), + `LONG_HIGH_VALUE` bigint(20), + `DOUBLE_HIGH_VALUE` double(53,4), + `DOUBLE_LOW_VALUE` double(53,4), + `BIG_DECIMAL_LOW_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `BIG_DECIMAL_HIGH_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `NUM_NULLS` bigint(20) NOT NULL, + `NUM_DISTINCTS` bigint(20), + `AVG_COL_LEN` double(53,4), + `MAX_COL_LEN` bigint(20), + `NUM_TRUES` bigint(20), + `NUM_FALSES` bigint(20), + `LAST_ANALYZED` bigint(20) NOT NULL, + PRIMARY KEY (`CS_ID`), + CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- +-- Table structure for table `PART_COL_STATS` +-- +CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( + `CS_ID` bigint(20) NOT NULL, + `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `TABLE_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `PART_ID` bigint(20) NOT NULL, + `LONG_LOW_VALUE` bigint(20), + `LONG_HIGH_VALUE` bigint(20), + `DOUBLE_HIGH_VALUE` double(53,4), + `DOUBLE_LOW_VALUE` double(53,4), + `BIG_DECIMAL_LOW_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `BIG_DECIMAL_HIGH_VALUE` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `NUM_NULLS` bigint(20) NOT NULL, + `NUM_DISTINCTS` bigint(20), + `AVG_COL_LEN` double(53,4), + `MAX_COL_LEN` bigint(20), + `NUM_TRUES` bigint(20), + `NUM_FALSES` bigint(20), + `LAST_ANALYZED` bigint(20) NOT NULL, + PRIMARY KEY (`CS_ID`), + CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME) USING BTREE; + +-- +-- Table structure for table `TYPES` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TYPES` ( + `TYPES_ID` bigint(20) NOT NULL, + `TYPE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TYPE1` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `TYPE2` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + PRIMARY KEY (`TYPES_ID`), + UNIQUE KEY `UNIQUE_TYPE` (`TYPE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `TYPE_FIELDS` +-- + +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE IF NOT EXISTS `TYPE_FIELDS` ( + `TYPE_NAME` bigint(20) NOT NULL, + `COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `FIELD_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `FIELD_TYPE` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + `INTEGER_IDX` int(11) NOT NULL, + PRIMARY KEY (`TYPE_NAME`,`FIELD_NAME`), + KEY `TYPE_FIELDS_N49` (`TYPE_NAME`), + CONSTRAINT `TYPE_FIELDS_FK1` FOREIGN KEY (`TYPE_NAME`) REFERENCES `TYPES` (`TYPES_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- Table `MASTER_KEYS` for classes [org.apache.hadoop.hive.metastore.model.MMasterKey] +CREATE TABLE IF NOT EXISTS `MASTER_KEYS` +( + `KEY_ID` INTEGER NOT NULL AUTO_INCREMENT, + `MASTER_KEY` VARCHAR(767) BINARY NULL, + PRIMARY KEY (`KEY_ID`) +) ENGINE=INNODB DEFAULT CHARSET=latin1; + +-- Table `DELEGATION_TOKENS` for classes [org.apache.hadoop.hive.metastore.model.MDelegationToken] +CREATE TABLE IF NOT EXISTS `DELEGATION_TOKENS` +( + `TOKEN_IDENT` VARCHAR(767) BINARY NOT NULL, + `TOKEN` VARCHAR(767) BINARY NULL, + PRIMARY KEY (`TOKEN_IDENT`) +) ENGINE=INNODB DEFAULT CHARSET=latin1; + +-- +-- Table structure for VERSION +-- +CREATE TABLE IF NOT EXISTS `VERSION` ( + `VER_ID` BIGINT NOT NULL, + `SCHEMA_VERSION` VARCHAR(127) NOT NULL, + `VERSION_COMMENT` VARCHAR(255), + PRIMARY KEY (`VER_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- +-- Table structure for table FUNCS +-- +CREATE TABLE IF NOT EXISTS `FUNCS` ( + `FUNC_ID` BIGINT(20) NOT NULL, + `CLASS_NAME` VARCHAR(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `CREATE_TIME` INT(11) NOT NULL, + `DB_ID` BIGINT(20), + `FUNC_NAME` VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin, + `FUNC_TYPE` INT(11) NOT NULL, + `OWNER_NAME` VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin, + `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin, + PRIMARY KEY (`FUNC_ID`), + UNIQUE KEY `UNIQUEFUNCTION` (`FUNC_NAME`, `DB_ID`), + KEY `FUNCS_N49` (`DB_ID`), + CONSTRAINT `FUNCS_FK1` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- +-- Table structure for table FUNC_RU +-- +CREATE TABLE IF NOT EXISTS `FUNC_RU` ( + `FUNC_ID` BIGINT(20) NOT NULL, + `RESOURCE_TYPE` INT(11) NOT NULL, + `RESOURCE_URI` VARCHAR(4000) CHARACTER SET latin1 COLLATE latin1_bin, + `INTEGER_IDX` INT(11) NOT NULL, + PRIMARY KEY (`FUNC_ID`, `INTEGER_IDX`), + CONSTRAINT `FUNC_RU_FK1` FOREIGN KEY (`FUNC_ID`) REFERENCES `FUNCS` (`FUNC_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE IF NOT EXISTS `NOTIFICATION_LOG` +( + `NL_ID` BIGINT(20) NOT NULL, + `EVENT_ID` BIGINT(20) NOT NULL, + `EVENT_TIME` INT(11) NOT NULL, + `EVENT_TYPE` varchar(32) NOT NULL, + `DB_NAME` varchar(128), + `TBL_NAME` varchar(256), + `MESSAGE` longtext, + `MESSAGE_FORMAT` varchar(16), + PRIMARY KEY (`NL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE IF NOT EXISTS `NOTIFICATION_SEQUENCE` +( + `NNI_ID` BIGINT(20) NOT NULL, + `NEXT_EVENT_ID` BIGINT(20) NOT NULL, + PRIMARY KEY (`NNI_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +( + `CHILD_CD_ID` BIGINT, + `CHILD_INTEGER_IDX` INT(11), + `CHILD_TBL_ID` BIGINT, + `PARENT_CD_ID` BIGINT NOT NULL, + `PARENT_INTEGER_IDX` INT(11) NOT NULL, + `PARENT_TBL_ID` BIGINT NOT NULL, + `POSITION` BIGINT NOT NULL, + `CONSTRAINT_NAME` VARCHAR(400) NOT NULL, + `CONSTRAINT_TYPE` SMALLINT(6) NOT NULL, + `UPDATE_RULE` SMALLINT(6), + `DELETE_RULE` SMALLINT(6), + `ENABLE_VALIDATE_RELY` SMALLINT(6) NOT NULL, + PRIMARY KEY (`CONSTRAINT_NAME`, `POSITION`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX `CONSTRAINTS_PARENT_TABLE_ID_INDEX` ON KEY_CONSTRAINTS (`PARENT_TBL_ID`) USING BTREE; + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL, + TXN_AGENT_INFO varchar(128), + TXN_META_INFO varchar(128), + TXN_HEARTBEAT_COUNT int +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint NOT NULL, + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128) NOT NULL, + TC_PARTITION varchar(767), + TC_OPERATION_TYPE char(1) NOT NULL, + FOREIGN KEY (TC_TXNID) REFERENCES TXNS (TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint NOT NULL, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(256), + CTC_PARTITION varchar(767) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767), + HL_LOCK_STATE char(1) not null, + HL_LOCK_TYPE char(1) not null, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + HL_HEARTBEAT_COUNT int, + HL_AGENT_INFO varchar(128), + HL_BLOCKEDBY_EXT_ID bigint, + HL_BLOCKEDBY_INT_ID bigint, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID), + KEY HIVE_LOCK_TXNID_INDEX (HL_TXNID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint, + CQ_META_INFO varbinary(2048), + CQ_HADOOP_JOB_ID varchar(32) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE COMPLETED_COMPACTIONS ( + CC_ID bigint PRIMARY KEY, + CC_DATABASE varchar(128) NOT NULL, + CC_TABLE varchar(128) NOT NULL, + CC_PARTITION varchar(767), + CC_STATE char(1) NOT NULL, + CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), + CC_WORKER_ID varchar(128), + CC_START bigint, + CC_END bigint, + CC_RUN_AS varchar(128), + CC_HIGHEST_TXN_ID bigint, + CC_META_INFO varbinary(2048), + CC_HADOOP_JOB_ID varchar(32) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + +CREATE TABLE AUX_TABLE ( + MT_KEY1 varchar(128) NOT NULL, + MT_KEY2 bigint NOT NULL, + MT_COMMENT varchar(255), + PRIMARY KEY(MT_KEY1, MT_KEY2) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE WRITE_SET ( + WS_DATABASE varchar(128) NOT NULL, + WS_TABLE varchar(128) NOT NULL, + WS_PARTITION varchar(767), + WS_TXNID bigint NOT NULL, + WS_COMMIT_ID bigint NOT NULL, + WS_OPERATION_TYPE char(1) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- ----------------------------------------------------------------- +-- Record schema version. Should be the last step in the init script +-- ----------------------------------------------------------------- +INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '2.3.0', 'Hive release version 2.3.0'); + +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-08-23 0:56:31 diff --git a/docs/database/submarine-data.sql b/docs/database/submarine-data.sql new file mode 100644 index 0000000000000000000000000000000000000000..93ed13d0d7ff400519ab36aedd4ec07122201cf9 --- /dev/null +++ b/docs/database/submarine-data.sql @@ -0,0 +1,96 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- http://www.apache.org/licenses/LICENSE-2.0 +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- ---------------------------- +-- Records of sys_dict +-- ---------------------------- +INSERT INTO `sys_dict` VALUES ('ca2dd544ca4c11e9a71e0242ac110002','SYS_USER_SEX','Sys user sex','submarine system dict, Do not modify.',0,0,NULL,'2019-08-29 11:04:36',NULL,'2019-09-01 01:08:12'); +INSERT INTO `sys_dict` VALUES ('f405a7b1cc5411e9af810242ac110002','SYS_USER_STATUS','Sys user status','submarine system dict, Do not modify.',0,0,NULL,'2019-09-01 01:08:05',NULL,'2019-09-01 01:08:05'); +INSERT INTO `sys_dict` VALUES ('3a1ed33ae83611e9ab840242ac110002','PROJECT_TYPE','Project machine learning type','submarine system dict, Do not modify.',0,0,NULL,'2019-09-01 01:08:05',NULL,'2019-09-01 01:08:05'); +INSERT INTO `sys_dict` VALUES ('8a101495e84011e9ab840242ac110002','PROJECT_VISIBILITY','Project visibility type','submarine system dict, Do not modify.',0,0,NULL,'2019-09-01 01:08:05',NULL,'2019-09-01 01:08:05'); +INSERT INTO `sys_dict` VALUES ('8f0439c9e84011e9ab840242ac110002','PROJECT_PERMISSION','Project permission type','submarine system dict, Do not modify.',0,0,NULL,'2019-09-01 01:08:05',NULL,'2019-09-01 01:08:05'); + +-- ---------------------------- +-- Records of sys_dict_item +-- ---------------------------- +INSERT INTO `sys_dict_item` VALUES ('27ef1080cc5511e9af810242ac110002','SYS_USER_STATUS_AVAILABLE','Available','SYS_USER_STATUS','submarine system dict, Do not modify.',1,0,NULL,'2019-09-01 01:09:32',NULL,'2019-09-01 01:13:19'); +INSERT INTO `sys_dict_item` VALUES ('4c5d736acc5511e9af810242ac110002','SYS_USER_STATUS_LOCKED','Locked','SYS_USER_STATUS','submarine system dict, Do not modify.',2,0,NULL,'2019-09-01 01:10:33',NULL,'2019-09-01 01:12:53'); +INSERT INTO `sys_dict_item` VALUES ('6d5ae3b2cc5511e9af810242ac110002','SYS_USER_STATUS_REGISTERED','New Registered','SYS_USER_STATUS','submarine system dict, Do not modify.',3,0,NULL,'2019-09-01 01:11:29',NULL,'2019-09-01 01:12:47'); +INSERT INTO `sys_dict_item` VALUES ('d018e2b0ca4c11e9a71e0242ac110002','SYS_USER_SEX_MALE','Male','SYS_USER_SEX','submarine system dict, Do not modify.',1,0,NULL,'2019-08-29 11:04:46',NULL,'2019-09-01 00:53:54'); +INSERT INTO `sys_dict_item` VALUES ('d94410adca4c11e9a71e0242ac110002','SYS_USER_SEX_FEMALE','Female','SYS_USER_SEX','submarine system dict, Do not modify.',2,0,NULL,'2019-08-29 11:05:02',NULL,'2019-09-01 00:54:00'); +INSERT INTO `sys_dict_item` VALUES ('7b9aafa0e83611e9ab840242ac110002','PROJECT_TYPE_NOTEBOOK','notebook','PROJECT_TYPE','submarine system dict, Do not modify.',1,0,NULL,'2019-08-29 11:05:02',NULL,'2019-09-01 00:54:00'); +INSERT INTO `sys_dict_item` VALUES ('8229a76be83611e9ab840242ac110002','PROJECT_TYPE_PYTHON','python','PROJECT_TYPE','submarine system dict, Do not modify.',2,0,NULL,'2019-08-29 11:05:02',NULL,'2019-09-01 00:54:00'); +INSERT INTO `sys_dict_item` VALUES ('ac80ab12e83611e9ab840242ac110002','PROJECT_TYPE_R','R','PROJECT_TYPE','submarine system dict, Do not modify.',3,0,NULL,'2019-08-29 11:05:02',NULL,'2019-09-01 00:54:00'); +INSERT INTO `sys_dict_item` VALUES ('b1070158e83611e9ab840242ac110002','PROJECT_TYPE_SCALA','scala','PROJECT_TYPE','submarine system dict, Do not modify.',4,0,NULL,'2019-08-29 11:05:02',NULL,'2019-09-01 00:54:00'); +INSERT INTO `sys_dict_item` VALUES ('8c53870be83611e9ab840242ac110002','PROJECT_TYPE_TENSORFLOW','tensorflow','PROJECT_TYPE','submarine system dict, Do not modify.',5,0,NULL,'2019-08-29 11:05:02',NULL,'2019-09-01 00:54:00'); +INSERT INTO `sys_dict_item` VALUES ('90ca63dfe83611e9ab840242ac110002','PROJECT_TYPE_PYTORCH','pytorch','PROJECT_TYPE','submarine system dict, Do not modify.',6,0,NULL,'2019-08-29 11:05:02',NULL,'2019-09-01 00:54:00'); +INSERT INTO `sys_dict_item` VALUES ('2ed844fae84111e9ab840242ac110002','PROJECT_VISIBILITY_PRIVATE','private','PROJECT_VISIBILITY','submarine system dict, Do not modify.',1,0,NULL,'2019-09-01 01:09:32',NULL,'2019-09-01 01:13:19'); +INSERT INTO `sys_dict_item` VALUES ('341d5a35e84111e9ab840242ac110002','PROJECT_VISIBILITY_TEAM','team','PROJECT_VISIBILITY','submarine system dict, Do not modify.',2,0,NULL,'2019-09-01 01:10:33',NULL,'2019-09-01 01:12:53'); +INSERT INTO `sys_dict_item` VALUES ('3866b369e84111e9ab840242ac110002','PROJECT_VISIBILITY_PUBLIC','public','PROJECT_VISIBILITY','submarine system dict, Do not modify.',3,0,NULL,'2019-09-01 01:11:29',NULL,'2019-09-01 01:12:47'); +INSERT INTO `sys_dict_item` VALUES ('3cc1a373e84111e9ab840242ac110002','PROJECT_PERMISSION_VIEW','can view','PROJECT_PERMISSION','submarine system dict, Do not modify.',1,0,NULL,'2019-09-01 01:09:32',NULL,'2019-09-01 01:13:19'); +INSERT INTO `sys_dict_item` VALUES ('44e90f6ce84111e9ab840242ac110002','PROJECT_PERMISSION_EDIT','can edit','PROJECT_PERMISSION','submarine system dict, Do not modify.',2,0,NULL,'2019-09-01 01:11:29',NULL,'2019-09-01 01:12:47'); +INSERT INTO `sys_dict_item` VALUES ('40dbb5ece84111e9ab840242ac110002','PROJECT_PERMISSION_EXECUTE','can execute','PROJECT_PERMISSION','submarine system dict, Do not modify.',3,0,NULL,'2019-09-01 01:10:33',NULL,'2019-09-01 01:12:53'); + +-- ---------------------------- +-- Records of sys_department +-- ---------------------------- +INSERT INTO `sys_department` VALUES ('e3d69d19c8d211e98edc0242ac110002','A','Company',NULL,0,'','0',NULL,'2019-08-27 13:59:30',NULL,'2019-08-27 14:02:05'); +INSERT INTO `sys_department` VALUES ('eec10fe9c8d211e98edc0242ac110002','AA','DepartmentA','A',0,'','0',NULL,'2019-08-27 13:59:48',NULL,'2019-08-27 14:04:11'); +INSERT INTO `sys_department` VALUES ('f8b42e19c8d211e98edc0242ac110002','AB','DepartmentB','A',0,'','0',NULL,'2019-08-27 14:00:05',NULL,'2019-08-27 14:07:19'); +INSERT INTO `sys_department` VALUES ('13a1916dc8d311e98edc0242ac110002','ABA','GroupA','AB',0,'','0',NULL,'2019-08-27 14:00:50',NULL,'2019-08-27 14:09:21'); +INSERT INTO `sys_department` VALUES ('1bc0cd98c8d311e98edc0242ac110002','AAA','GroupB','AA',0,'','0',NULL,'2019-08-27 14:01:04',NULL,'2019-08-29 16:48:56'); + +-- ---------------------------- +-- Records of sys_user +-- ---------------------------- +INSERT INTO `sys_user` VALUES ('e9ca23d68d884d4ebb19d07889727dae', 'admin', 'administrator', '21232f297a57a5a743894a0e4a801fc3', 'avatar.png', '2018-12-05 00:00:00', NULL, 'dev@submarine.org', '18566666661', NULL, NULL, NULL, 1, 'admin', '2019-07-05 14:47:22', 'admin', '2019-07-05 14:47:22'); + +-- ---------------------------- +-- Records of team +-- ---------------------------- +INSERT INTO `team` VALUES ('e9ca23d68d884d4ebb19d07889721234', 'admin', 'Submarine', 'admin', '2020-05-06 14:00:05', 'Jack', '2020-05-06 14:00:14'); + +-- ---------------------------- +-- Records of metrics +-- ---------------------------- +INSERT INTO `metrics` (`id`, `key`, `value`, `worker_index`, `timestamp`, `step`, `is_nan`) VALUES +('application_123651651', 'score', 0.666667, 'worker-1', 1569139525097, 0, 0), +('application_123651651', 'score', 0.666670, 'worker-1', 1569149139731, 1, 0), +('experiment_1595332719154_0001', 'score', 0.666667, 'worker-1', 1569169376482, 0, 0), +('experiment_1595332719154_0001', 'score', 0.666671, 'worker-1', 1569236290721, 0, 0), +('experiment_1595332719154_0001', 'score', 0.666680, 'worker-1', 1569236466722, 0, 0); + +-- ---------------------------- +-- Records of params +-- ---------------------------- +INSERT INTO `params` (`id`, `key`, `value`, `worker_index`) VALUES +('application_123651651', 'max_iter', '100', 'worker-1'), +('application_123456898', 'n_jobs', '5', 'worker-1'), +('application_123456789', 'alpha', '20', 'worker-1'), +('experiment_1595332719154_0001', 'max_iter', '100', 'worker-1'), +('experiment_1595332719154_0002', 'n_jobs', '5', 'worker-1'), +('experiment_1595332719154_0003', 'alpha', '20', 'worker-1'); + +-- ---------------------------- +-- Records of environment +-- ---------------------------- +INSERT INTO `environment` VALUES +('environment_1600862964725_0001', 'notebook-env', '{"name":"notebook-env","dockerImage":"apache/submarine:jupyter-notebook-0.5.0","kernelSpec":{"name":"submarine_jupyter_py3","channels":["defaults"],"dependencies":[]}}', 'admin', '2020-09-21 14:00:05', 'admin', '2020-09-21 14:00:14'); + +-- ---------------------------- +-- Records of experiment_templates +-- ---------------------------- +INSERT INTO `experiment_template` (`id`, `experimentTemplate_name`, `experimentTemplate_spec`, `create_by`, `create_time`, `update_by`, `update_time`) VALUES +('experimentTemplate_1599498007985_0013', 'tf-mnist', '{\"name\": \"tf-mnist\", \"author\": \"author\", \"parameters\": [{\"name\": \"learning_rate\", \"value\": \"0.2\", \"required\": \"false\", \"description\": \"The parameter of train mnist.\"}, {\"name\": \"batch_size\", \"value\": \"150\", \"required\": \"false\", \"description\": \"The parameter of train mnist.\"}, {\"name\": \"experiment_name\", \"required\": \"true\", \"description\": \"experiment name, you should change it to avoid duplication with other experiment names.\"}, {\"name\": \"spec.Ps.replicas\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Ps.resourceMap.cpu\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Ps.resourceMap.memory\", \"value\": \"1000M\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Worker.replicas\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Worker.resourceMap.cpu\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Worker.resourceMap.memory\", \"value\": \"1000M\", \"required\": \"false\", \"description\": \"\"}], \"description\": \"This is a template to run tf-mnist.\", \"experimentSpec\": {\"meta\": {\"cmd\": \"python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate={{learning_rate}} --batch_size={{batch_size}}\", \"name\": \"{{experiment_name}}\", \"envVars\": {\"ENV1\": \"ENV1\"}, \"framework\": \"TensorFlow\", \"namespace\": \"default\"}, \"spec\": {\"Ps\": {\"replicas\": 1, \"resources\": \"cpu=1,memory=1000M\", \"resourceMap\": {\"cpu\": \"1\", \"memory\": \"1000M\"}}, \"Worker\": {\"replicas\": 1, \"resources\": \"cpu=1,memory=1000M\", \"resourceMap\": {\"cpu\": \"1\", \"memory\": \"1000M\"}}}, \"environment\": {\"image\": \"apache/submarine:tf-mnist-with-summaries-1.0\"}}}', NULL, '2020-09-10 16:31:32', NULL, '2020-10-19 17:05:21'); + +INSERT INTO `experiment_template` (`id`, `experimentTemplate_name`, `experimentTemplate_spec`, `create_by`, `create_time`, `update_by`, `update_time`) VALUES('experimentTemplate_1606489231336_0014', 'pytorch-mnist', '{\"name\": \"pytorch-mnist\", \"author\": \"author\", \"parameters\": [{\"name\": \"experiment_name\", \"required\": \"true\", \"description\": \"experiment name\"}, {\"name\": \"spec.Master.replicas\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Master.resourceMap.cpu\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Master.resourceMap.memory\", \"value\": \"1024M\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Worker.replicas\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Worker.resourceMap.cpu\", \"value\": \"1\", \"required\": \"false\", \"description\": \"\"}, {\"name\": \"spec.Worker.resourceMap.memory\", \"value\": \"1024M\", \"required\": \"false\", \"description\": \"\"}], \"description\": \"This is a template to run pytorch-mnist\\n\", \"experimentSpec\": {\"meta\": {\"cmd\": \"python /var/mnist.py --backend gloo\", \"name\": \"{{experiment_name}}\", \"envVars\": {\"ENV_1\": \"ENV1\"}, \"framework\": \"PyTorch\", \"namespace\": \"default\"}, \"spec\": {\"Master\": {\"replicas\": 1, \"resources\": \"cpu=1,memory=1024M\", \"resourceMap\": {\"cpu\": \"1\", \"memory\": \"1024M\"}}, \"Worker\": {\"replicas\": 1, \"resources\": \"cpu=1,memory=1024M\", \"resourceMap\": {\"cpu\": \"1\", \"memory\": \"1024M\"}}}, \"environment\": {\"image\": \"apache/submarine:pytorch-dist-mnist-1.0\"}}}', NULL, '2020-11-29 17:56:10', NULL, '2020-11-29 17:56:10'); diff --git a/docs/database/submarine.sql b/docs/database/submarine.sql new file mode 100644 index 0000000000000000000000000000000000000000..76dfc490ed709b6b2f2ec0b9b6107f14e6a3a194 --- /dev/null +++ b/docs/database/submarine.sql @@ -0,0 +1,281 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- http://www.apache.org/licenses/LICENSE-2.0 +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- ---------------------------- +-- Table structure for sys_dict +-- ---------------------------- +DROP TABLE IF EXISTS `sys_dict`; +CREATE TABLE `sys_dict` ( + `id` varchar(32) NOT NULL, + `dict_code` varchar(32) NOT NULL COMMENT 'dict code', + `dict_name` varchar(32) NOT NULL COMMENT 'dict name', + `description` varchar(255) default NULL COMMENT 'dict description', + `deleted` int(1) default 0 COMMENT 'delete status(0:normal, 1:already deleted)', + `type` int(1) default 0 COMMENT 'dict type (0:string,1:number)', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`), + UNIQUE KEY `UK_SYS_DICT_DICT_CODE` (`dict_code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for sys_dict_item +-- ---------------------------- +DROP TABLE IF EXISTS `sys_dict_item`; +CREATE TABLE `sys_dict_item` ( + `id` varchar(32) NOT NULL, + `item_code` varchar(32) NOT NULL COMMENT 'dict item code', + `item_name` varchar(32) NOT NULL COMMENT 'dict item name', + `dict_code` varchar(32) NOT NULL COMMENT 'dict code', + `description` varchar(255) default NULL COMMENT 'description', + `sort_order` int(3) default 0 COMMENT 'sort order', + `deleted` int(1) default 0 COMMENT 'delete status(0:normal,1:already deleted)', + `create_by` varchar(32) default NULL, + `create_time` datetime default NULL, + `update_by` varchar(32) default NULL, + `update_time` datetime default NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `UK_SYS_DICT_ITEM_CODE` (`item_code`)/*, + CONSTRAINT `FK_SYS_DICT_ITEM_DICT_CODE` FOREIGN KEY (`dict_code`) REFERENCES `sys_dict` (`dict_code`)*/ +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for system department +-- ---------------------------- +DROP TABLE IF EXISTS `sys_department`; +CREATE TABLE `sys_department` ( + `id` varchar(32) NOT NULL COMMENT 'ID', + `dept_code` varchar(32) NOT NULL COMMENT 'department code', + `dept_name` varchar(64) NOT NULL COMMENT 'department name', + `parent_code` varchar(32) default NULL COMMENT 'parent dept code', + `sort_order` int(3) default 0 COMMENT 'sort order', + `description` varchar(255) COMMENT 'description', + `deleted` varchar(1) default 0 COMMENT 'delete status(0:normal,1:already deleted)', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`), + UNIQUE KEY `UK_DEPT_CODE` (`dept_code`)/*, + CONSTRAINT `FK_SYS_DEPT_PARENT_CODE` FOREIGN KEY (`parent_code`) REFERENCES `sys_department` (`dept_code`)*/ +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for sys_user +-- ---------------------------- +DROP TABLE IF EXISTS `sys_user`; +CREATE TABLE `sys_user` ( + `id` varchar(32) NOT NULL COMMENT 'id', + `user_name` varchar(32) NOT NULL COMMENT 'login name', + `real_name` varchar(64) NOT NULL COMMENT 'real name', + `password` varchar(255) NOT NULL COMMENT 'password', + `avatar` varchar(255) default NULL COMMENT 'avatar', + `birthday` datetime default NULL COMMENT 'birthday', + `sex` varchar(32) default NULL COMMENT 'sex', + `email` varchar(32) default NULL COMMENT 'email', + `phone` varchar(32) default NULL COMMENT 'telphone', + `dept_code` varchar(32) default NULL COMMENT 'department code', + `role_code` varchar(32) default NULL COMMENT 'role code', + `status` varchar(32) default NULL COMMENT 'status', + `deleted` int(1) default 0 COMMENT 'deleted status(0:normal, 1:already deleted)', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`), + UNIQUE KEY `sys_user_name` (`user_name`)/*, + CONSTRAINT `FK_SYS_USER_DEPT_CODE` FOREIGN KEY (`dept_code`) REFERENCES `sys_department` (`dept_code`), + CONSTRAINT `FK_SYS_USER_SEX` FOREIGN KEY (`sex`) REFERENCES `sys_dict_item` (`item_code`), + CONSTRAINT `FK_SYS_USER_STATUS` FOREIGN KEY (`status`) REFERENCES `sys_dict_item` (`item_code`)*/ +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for sys_message +-- ---------------------------- +DROP TABLE IF EXISTS `sys_message`; +CREATE TABLE `sys_message` ( + `id` varchar(32) NOT NULL COMMENT 'id', + `sender` varchar(32) default NULL COMMENT 'sender user', + `receiver` varchar(32) default NULL COMMENT 'receiver user', + `type` varchar(32) default NULL COMMENT 'dict_code:MESSAGE_TYPE', + `context` text COMMENT 'message context', + `status` int(1) default 0 COMMENT '0:unread, 1:read', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`)/*, + CONSTRAINT `FK_SYS_MSG_SENDER` FOREIGN KEY (`sender`) REFERENCES `sys_user` (`user_name`), + CONSTRAINT `FK_SYS_MSG_RECEIVER` FOREIGN KEY (`receiver`) REFERENCES `sys_user` (`user_name`), + CONSTRAINT `FK_SYS_MSG_TYPE` FOREIGN KEY (`type`) REFERENCES `sys_dict_item` (`item_code`)*/ +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for team +-- ---------------------------- +DROP TABLE IF EXISTS `team`; +CREATE TABLE `team` ( + `id` varchar(32) NOT NULL, + `owner` varchar(100) NOT NULL COMMENT 'owner name', + `team_name` varchar(64) NOT NULL COMMENT 'team name', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`), + UNIQUE KEY `UK_TEAM_NAME` (`team_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for team_member +-- ---------------------------- +DROP TABLE IF EXISTS `team_member`; +CREATE TABLE `team_member` ( + `id` varchar(32) NOT NULL, + `team_id` varchar(32) NOT NULL COMMENT 'team id', + `team_name` varchar(64) NOT NULL COMMENT 'team name', + `member` varchar(100) NOT NULL COMMENT 'member name', + `inviter` int(1) default 0 COMMENT '0:inviter, 1:accept', + `create_by` varchar(32) default NULL, + `create_time` datetime default NULL, + `update_by` varchar(32) default NULL, + `update_time` datetime default NULL, + PRIMARY KEY (`id`)/*, + CONSTRAINT `FK_TEAM_MEMBER_USER` FOREIGN KEY (`member`) REFERENCES `sys_user` (`user_name`)*/ +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for project +-- ---------------------------- +DROP TABLE IF EXISTS `project`; +CREATE TABLE `project` ( + `id` varchar(32) NOT NULL, + `name` varchar(100) NOT NULL COMMENT 'project name', + `visibility` varchar(32) default NULL COMMENT 'dict_code:PROJECT_VISIBILITY', + `permission` varchar(32) default NULL COMMENT 'dict_code:PROJECT_PERMISSION', + `type` varchar(32) default NULL COMMENT 'dict_code:PROJECT_TYPE', + `description` varchar(255) COMMENT 'description', + `user_name` varchar(32) NOT NULL COMMENT 'owner user id', + `team_name` varchar(32) default NULL COMMENT 'team name', + `tags` varchar(128) default NULL COMMENT 'Comma separated tag', + `star_num` int(8) default 0 COMMENT 'star number', + `like_num` int(8) default 0 COMMENT 'like number', + `message_num` int(8) default 0 COMMENT 'message number', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`)/*, + CONSTRAINT `FK_PROJECT_TYPE` FOREIGN KEY (`type`) REFERENCES `sys_dict_item` (`item_code`), + CONSTRAINT `FK_PROJECT_TEAM_NAME` FOREIGN KEY (`team_name`) REFERENCES `team` (`team_name`), + CONSTRAINT `FK_PROJECT_USER_NAME` FOREIGN KEY (`user_name`) REFERENCES `sys_user` (`user_name`)*/ +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for project_files +-- ---------------------------- +DROP TABLE IF EXISTS `project_files`; +CREATE TABLE `project_files` ( + `id` varchar(32) NOT NULL, + `project_id` varchar(32) NOT NULL COMMENT 'project id', + `file_name` varchar(128) NOT NULL COMMENT '/path/.../file.suffix', + `file_content` text default NULL COMMENT 'file content', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`)/*, + CONSTRAINT `FK_PROJECT_FILES_PRJ_ID` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`)*/ +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for jobs +-- ---------------------------- +DROP TABLE IF EXISTS `job`; +CREATE TABLE `job` ( + `id` int NOT NULL AUTO_INCREMENT, + `job_id` varchar(64) default NULL COMMENT 'job id', + `job_name` varchar(64) NOT NULL COMMENT 'job name', + `job_type` varchar(64) NOT NULL COMMENT 'job type', + `job_namespace` varchar(32) default NULL COMMENT 'job namespace', + `job_status` varchar(32) default NULL COMMENT 'job status', + `job_final_status` varchar(32) default NULL COMMENT 'job final status', + `user_name` varchar(32) default NULL COMMENT 'user name', + `create_by` varchar(32) default NULL COMMENT 'create user', + `create_time` datetime default NULL COMMENT 'create time', + `update_by` varchar(32) default NULL COMMENT 'last update user', + `update_time` datetime default NULL COMMENT 'last update time', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for environment +-- ---------------------------- +DROP TABLE IF EXISTS `environment`; +CREATE TABLE `environment` ( + `id` varchar(64) NOT NULL COMMENT 'Id of the Environment', + `environment_name` varchar(255) NOT NULL COMMENT 'Name of the Environment', + `environment_spec` text NOT NULL COMMENT 'Spec of the Environment', + `create_by` varchar(32) DEFAULT NULL COMMENT 'create user', + `create_time` datetime DEFAULT NULL COMMENT 'create time', + `update_by` varchar(32) DEFAULT NULL COMMENT 'last update user', + `update_time` datetime DEFAULT NULL COMMENT 'last update time', + PRIMARY KEY `id` (`id`), + UNIQUE KEY `environment_name` (`environment_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for metric +-- ---------------------------- +DROP TABLE IF EXISTS `metrics`; +CREATE TABLE `metrics` ( + `id` varchar(64) NOT NULL COMMENT 'Id of the Experiment', + `key` varchar(190) NOT NULL COMMENT 'Metric key: `String` (limit 190 characters). Part of *Primary Key* for ``metrics`` table.', + `value` float NOT NULL COMMENT 'Metric value: `Float`. Defined as *Non-null* in schema.', + `worker_index` varchar(32) NOT NULL COMMENT 'Metric worker_index: `String` (limit 32 characters). Part of *Primary Key* for\r\n ``metrics`` table.', + `timestamp` bigint(20) NOT NULL COMMENT 'Timestamp recorded for this metric entry: `BigInteger`. Part of *Primary Key* for ``metrics`` table.', + `step` bigint(11) NOT NULL COMMENT 'Step recorded for this metric entry: `BigInteger`.', + `is_nan` BOOLEAN NOT NULL COMMENT 'True if the value is in fact NaN.', + PRIMARY KEY (`id`, `key`, `timestamp`, `worker_index`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for param +-- ---------------------------- +DROP TABLE IF EXISTS `params`; +CREATE TABLE `params` ( + `id` varchar(64) NOT NULL COMMENT 'Id of the Experiment', + `key` varchar(190) NOT NULL COMMENT '`String` (limit 190 characters). Part of *Primary Key* for ``params`` table.', + `value` varchar(32) NOT NULL COMMENT '`String` (limit 190 characters). Defined as *Non-null* in schema.', + `worker_index` varchar(32) NOT NULL COMMENT '`String` (limit 32 characters). Part of *Primary Key* for\r\n ``metrics`` table.', + PRIMARY KEY (`id`, `key`, `worker_index`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +-- ---------------------------- +-- Table structure for experiment_templates +-- ---------------------------- +DROP TABLE IF EXISTS `experiment_template`; +CREATE TABLE `experiment_template` ( + `id` varchar(64) NOT NULL, + `experimentTemplate_name` varchar(32) NOT NULL, + `experimentTemplate_spec` json DEFAULT NULL, + `create_by` varchar(32) DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_by` varchar(32) DEFAULT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY `id` (`id`), + UNIQUE KEY `experimentTemplate_name` (`experimentTemplate_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/docs/design/architecture-and-requirements.md b/docs/design/architecture-and-requirements.md new file mode 100644 index 0000000000000000000000000000000000000000..d41e8795a8ade0c1d9301cf8a687fcde33d3ad44 --- /dev/null +++ b/docs/design/architecture-and-requirements.md @@ -0,0 +1,323 @@ + + +# Terminology + +| Term | Description | +| -------- | -------- | +| User | A single data-scientist/data-engineer. User has resource quota, credentials | +| Team | User belongs to one or more teams, teams have ACLs for artifacts sharing such as notebook content, model, etc. | +| Admin | Also called SRE, who manages user's quotas, credentials, team, and other components. | + + +# Background + +Everybody talks about machine learning today, and lots of companies are trying to leverage machine learning to push the business to the next level. Nowadays, as more and more developers, infrastructure software companies coming to this field, machine learning becomes more and more achievable. + +In the last decade, the software industry has built many open source tools for machine learning to solve the pain points: + +1. It was not easy to build machine learning algorithms manually, such as logistic regression, GBDT, and many other algorithms: + **Answer to that:** Industries have open sourced many algorithm libraries, tools, and even pre-trained models so that data scientists can directly reuse these building blocks to hook up to their data without knowing intricate details inside these algorithms and models. + +2. It was not easy to achieve "WYSIWYG, what you see is what you get" from IDEs: not easy to get output, visualization, troubleshooting experiences at the same place. + **Answer to that:** Notebooks concept was added to this picture, notebook brought the experiences of interactive coding, sharing, visualization, debugging under the same user interface. There're popular open-source notebooks like Apache Zeppelin/Jupyter. + +3. It was not easy to manage dependencies: ML applications can run on one machine is hard to deploy on another machine because it has lots of libraries dependencies. + **Answer to that:** Containerization becomes popular and a standard to packaging dependencies to make it easier to "build once, run anywhere". + +4. Fragmented tools, libraries were hard for ML engineers to learn. Experiences learned in one company are not naturally migratable to another company. + **Answer to that:** A few dominant open-source frameworks reduced the overhead of learning too many different frameworks, concepts. Data-scientist can learn a few libraries such as Tensorflow/PyTorch, and a few high-level wrappers like Keras will be able to create your machine learning application from other open-source building blocks. + +5. Similarly, models built by one library (such as libsvm) were hard to be integrated into machine learning pipeline since there's no standard format. + **Answer to that:** Industry has built successful open-source standard machine learning frameworks such as Tensorflow/PyTorch/Keras so their format can be easily shared across. And efforts to build an even more general model format such as ONNX. + +6. It was hard to build a data pipeline that flows/transform data from a raw data source to whatever required by ML applications. + **Answer to that:** Open source big data industry plays an important role in providing, simplify, unify processes and building blocks for data flows, transformations, etc. + +The machine learning industry is moving on the right track to solve major roadblocks. So what are the pain points now for companies which have machine learning needs? What can we help here? To answer this question, let's look at machine learning workflow first. + +## Machine Learning Workflows & Pain points + +``` +1) From different data sources such as edge, clickstream, logs, etc. + => Land to data lakes + +2) From data lake, data transformation: + => Data transformations: Cleanup, remove invalid rows/columns, + select columns, sampling, split train/test + data-set, join table, etc. + => Data prepared for training. + +3) From prepared data: + => Training, model hyper-parameter tuning, cross-validation, etc. + => Models saved to storage. + +4) From saved models: + => Model assurance, deployment, A/B testing, etc. + => Model deployed for online serving or offline scoring. +``` + +Typically data scientists responsible for item 2)-4), 1) typically handled by a different team (called Data Engineering team in many companies, some Data Engineering team also responsible for part of data transformation) + +### Pain \#1 Complex workflow/steps from raw data to model, different tools needed by different steps, hard to make changes to workflow, and not error-proof + +It is a complex workflow from raw data to usable models, after talking to many different data scientists, we have learned that a typical procedure to train a new model and push to production can take months to 1-2 years. + +It is also a wide skill set required by this workflow. For example, data transformation needs tools like Spark/Hive for large scale and tools like Pandas for a small scale. And model training needs to be switched between XGBoost, Tensorflow, Keras, PyTorch. Building a data pipeline requires Apache Airflow or Oozie. + +Yes, there are great, standardized open-source tools built for many of such purposes. But how about changes need to be made for a particular part of the data pipeline? How about adding a few columns to the training data for experiments? How about training models, and push models to validation, A/B testing before rolling to production? All these steps need jumping between different tools, UIs, and very hard to make changes, and it is not error-proof during these procedures. + +### Pain \#2 Dependencies of underlying resource management platform + +To make jobs/services required by a machine learning platform to be able to run, we need an underlying resource management platform. There're some choices of resource management platform, and they have distinct advantages and disadvantages. + +For example, there're many machine learning platform built on top of K8s. It is relatively easy to get a K8s from a cloud vendor, easy to orchestrate machine learning required services/daemons run on K8s. However, K8s doesn't offer good support jobs like Spark/Flink/Hive. So if your company has Spark/Flink/Hive running on YARN, there're gaps and a significant amount of work to move required jobs from YARN to K8s. Maintaining a separate K8s cluster is also overhead to Hadoop-based data infrastructure. + +Similarly, if your company's data pipelines are mostly built on top of cloud resources and SaaS offerings, asking you to install a separate YARN cluster to run a new machine learning platform doesn't make a lot of sense. + +### Pain \#3 Data scientist are forced to interact with lower-level platform components + +In addition to the above pain, we do see Data Scientists are forced to learn underlying platform knowledge to be able to build a real-world machine learning workflow. + +For most of the data scientists we talked with, they're experts of ML algorithms/libraries, feature engineering, etc. They're also most familiar with Python, R, and some of them understand Spark, Hive, etc. + +If they're asked to do interactions with lower-level components like fine-tuning a Spark job's performance; or troubleshooting job failed to launch because of resource constraints; or write a K8s/YARN job spec and mount volumes, set networks properly. They will scratch their heads and typically cannot perform these operations efficiently. + +### Pain \#4 Comply with data security/governance requirements + +TODO: Add more details. + +### Pain \#5 No good way to reduce routine ML code development + +After the data is prepared, the data scientist needs to do several routine tasks to build the ML pipeline. To get a sense of the existing the data set, it usually needs a split of the data set, the statistics of data set. These tasks have a common duplicate part of code, which reduces the efficiency of data scientists. + +An abstraction layer/framework to help the developer to boost ML pipeline development could be valuable. It's better than the developer only needs to fill callback function to focus on their key logic. + +# Submarine + +## Overview + +### A little bit history + +Initially, Submarine is built to solve problems of running deep learning jobs like Tensorflow/PyTorch on Apache Hadoop YARN, allows admin to monitor launched deep learning jobs, and manage generated models. + +It was part of YARN initially, and code resides under `hadoop-yarn-applications`. Later, the community decided to convert it to be a subproject within Hadoop (Sibling project of YARN, HDFS, etc.) because we want to support other resource management platforms like K8s. And finally, we're reconsidering Submarine's charter, and the Hadoop community voted that it is the time to moved Submarine to a separate Apache TLP. + +### Why Submarine? + +`ONE PLATFORM` + +Submarine is the ONE PLATFORM to allow Data Scientists to create end-to-end machine learning workflow. `ONE PLATFORM` means it supports Data Scientists and data engineers to finish their jobs on the same platform without frequently switching their toolsets. From dataset exploring data pipeline creation, model training, and tuning, and push model to production. All these steps can be completed within the `ONE PLATFORM`. + +`Resource Management Independent` + +It is also designed to be resource management independent, no matter if you have Apache Hadoop YARN, K8s, or just a container service, you will be able to run Submarine on top it. + + +## Requirements and non-requirements + +### Notebook + +1) Users should be able to create, edit, delete a notebook. (P0) +2) Notebooks can be persisted to storage and can be recovered if failure happens. (P0) +3) Users can trace back to history versions of a notebook. (P1) +4) Notebooks can be shared with different users. (P1) +5) Users can define a list of parameters of a notebook (looks like parameters of the notebook's main function) to allow executing a notebook like a job. (P1) +6) Different users can collaborate on the same notebook at the same time. (P2) + +A running notebook instance is called notebook session (or session for short). + +### Experiment + +Experiments of Submarine is an offline task. It could be a shell command, a Python command, a Spark job, a SQL query, or even a workflow. + +The primary purposes of experiments under Submarine's context is to do training tasks, offline scoring, etc. However, experiment can be generalized to do other tasks as well. + +Major requirement of experiment: + +1) Experiments can be submitted from UI/CLI/SDK. +2) Experiments can be monitored/managed from UI/CLI/SDK. +3) Experiments should not bind to one resource management platform (K8s/YARN). + +#### Type of experiments + +![](../assets/design/experiments.png) + +There're two types of experiments: +`Adhoc experiments`: which includes a Python/R/notebook, or even an adhoc Tensorflow/PyTorch task, etc. + +`Predefined experiment library`: This is specialized experiments, which including developed libraries such as CTR, BERT, etc. Users are only required to specify a few parameters such as input, output, hyper parameters, etc. Instead of worrying about where's training script/dependencies located. + +#### Adhoc experiment + +Requirements: + +- Allow run adhoc scripts. +- Allow model engineer, data scientist to run Tensorflow/Pytorch programs on YARN/K8s/Container-cloud. +- Allow jobs easy access data/models in HDFS/s3, etc. +- Support run distributed Tensorflow/Pytorch jobs with simple configs. +- Support run user-specified Docker images. +- Support specify GPU and other resources. + +#### Predefined experiment library + +Here's an example of predefined experiment library to train deepfm model: + +``` +{ + "input": { + "train_data": ["hdfs:///user/submarine/data/tr.libsvm"], + "valid_data": ["hdfs:///user/submarine/data/va.libsvm"], + "test_data": ["hdfs:///user/submarine/data/te.libsvm"], + "type": "libsvm" + }, + "output": { + "save_model_dir": "hdfs:///user/submarine/deepfm", + "metric": "auc" + }, + "training": { + "batch_size" : 512, + "field_size": 39, + "num_epochs": 3, + "feature_size": 117581, + ... + } +} +``` + +Predefined experiment libraries can be shared across users on the same platform, users can also add new or modified predefined experiment library via UI/REST API. + +We will also model AutoML, auto hyper-parameter tuning to predefined experiment library. + +#### Pipeline + +Pipeline is a special kind of experiment: + +- A pipeline is a DAG of experiments. +- Can be also treated as a special kind of experiment. +- Users can submit/terminate a pipeline. +- Pipeline can be created/submitted via UI/API. + +### Environment Profiles + +Environment profiles (or environment for short) defines a set of libraries and when Docker is being used, a Docker image in order to run an experiment or a notebook. + +Docker or VM image (such as AMI: Amazon Machine Images) defines the base layer of the environment. + +On top of that, users can define a set of libraries (such as Python/R) to install. + +Users can save different environment configs which can be also shared across the platform. Environment profiles can be used to run a notebook (e.g. by choosing different kernel from Jupyter), or an experiment. Predefined experiment library includes what environment to use so users don't have to choose which environment to use. + +Environments can be added/listed/deleted/selected through CLI/SDK. + +### Model + +#### Model management + +- Model artifacts are generated by experiments or notebook. +- A model consists of artifacts from one or multiple files. +- Users can choose to save, tag, version a produced model. +- Once The Model is saved, Users can do the online model serving or offline scoring of the model. + +#### Model serving + +After model saved, users can specify a serving script, a model and create a web service to serve the model. + +We call the web service to "endpoint". Users can manage (add/stop) model serving endpoints via CLI/API/UI. + +### Metrics for training job and model + +Submarine-SDK provides tracking/metrics APIs, which allows developers to add tracking/metrics and view tracking/metrics from Submarine Workbench UI. + +### Deployment + +Submarine Services (See architecture overview below) should be deployed easily on-prem / on-cloud. Since there're more and more public cloud offering for compute/storage management on cloud, we need to support deploy Submarine compute-related workloads (such as notebook session, experiments, etc.) to cloud-managed clusters. + +This also include Submarine may need to take input parameters from customers and create/manage clusters if needed. It is also a common requirement to use hybrid of on-prem/on-cloud clusters. + +### Security / Access Control / User Management / Quota Management + +There're 4 kinds of objects need access-control: + +- Assets belong to Submarine system, which includes notebook, experiments and results, models, predefined experiment libraries, environment profiles. +- Data security. (Who owns what data, and what data can be accessed by each users). +- User credentials. (Such as LDAP). +- Other security, such as Git repo access, etc. + +For the data security / user credentials / other security, it will be delegated to 3rd libraries such as Apache Ranger, IAM roles, etc. + +Assets belong to Submarine system will be handled by Submarine itself. + +Here're operations which Submarine admin can do for users / teams which can be used to access Submarine's assets. + +**Operations for admins** + +- Admin uses "User Management System" to onboard new users, upload user credentials, assign resource quotas, etc. +- Admins can create new users, new teams, update user/team mappings. Or remove users/teams. +- Admin can set resource quotas (if different from system default), permissions, upload/update necessary credentials (like Kerberos keytab) of a user. +- A DE/DS can also be an admin if the DE/DS has admin access. (Like a privileged user). This will be useful when a cluster is exclusively shared by a user or only shared by a small team. +- `Resource Quota Management System` helps admin to manage resources quotas of teams, organizations. Resources can be machine resources like CPU/Memory/Disk, etc. It can also include non-machine resources like $$-based budgets. + +### Dataset + +There's also need to tag dataset which will be used for training and shared across the platform by different users. + +Like mentioned above, access to the actual data will be handled by 3rd party system like Apache Ranger / Hive Metastore which is out of the Submarine's scope. + +## Architecture Overview + +### Architecture Diagram + +``` + +-----------------------------------------------------------------+ + | Submarine UI / CLI / REST API / SDK | + | Mini-Submarine | + +-----------------------------------------------------------------+ + + +--------------------Submarine Server-----------------------------+ + | +---------+ +---------+ +----------+ +----------+ +------------+| + | |Data set | |Notebooks| |Experiment| |Models | |Servings || + | +---------+ +---------+ +----------+ +----------+ +------------+| + |-----------------------------------------------------------------| + | | + | +-----------------+ +-----------------+ +---------------------+ | + | |Experiment | |Compute Resource | |Other Management | | + | |Manager | | Manager | |Services | | + | +-----------------+ +-----------------+ +---------------------+ | + | Spark, template YARN/K8s/Docker | + | TF, PyTorch, pipeline | + | | + + +-----------------+ + + | |Submarine Meta | | + | | Store | | + | +-----------------+ | + | | + +-----------------------------------------------------------------+ + + (You can use http://stable.ascii-flow.appspot.com/#Draw + to draw such diagrams) +``` + +`Compute Resource Manager` Helps to manage compute resources on-prem/on-cloud, this module can also handle cluster creation / management, etc. + +`Experiment Manager` Work with "Compute Resource Manager" to submit different kinds of workloads such as (distributed) Tensorflow / Pytorch, etc. + +`Submarine SDK` provides Java/Python/REST API to allow DS or other engineers to integrate into Submarine services. It also includes a `mini-submarine` component that launches Submarine components from a single Docker container (or a VM image). + +Details of Submarine Server design can be found at [submarine-server-design](./submarine-server/architecture.md). + +# References + + diff --git a/docs/design/environments-implementation.md b/docs/design/environments-implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..14e51c8b74f34cdc59a6a016414952e0547da60c --- /dev/null +++ b/docs/design/environments-implementation.md @@ -0,0 +1,194 @@ + + +# Overview + +Environment profiles (or environment for short) defines a set of libraries and when Docker is being used, a Docker image in order to run an experiment or a notebook. + +Docker and/or VM-image (such as, VirtualBox/VMWare images, Amazon Machine Images - AMI, Or custom image of Azure VM) defines the base layer of the environment. Please note that VM-image is different from VM instance type, + +On top of that, users can define a set of libraries (such as Python/R) to install, we call it kernel. + +**Example of Environment** + +``` + + +-------------------+ + |+-----------------+| + || Python=3.7 || + || Tensorflow=2.0 || + |+---Exp Dependency+| + |+-----------------+| + ||OS=Ubuntu16.04 || + ||CUDA=10.2 || + ||GPU_Driver=375.. || + |+---Base Library--+| + +-------------------+ +``` + +As you can see, There're base libraries, such as what OS, CUDA version, GPU driver, etc. They can be achieved by specifying a VM-image / Docker image. + +On top of that, user can bring their dependencies, such as different version of Python, Tensorflow, Pandas, etc. + +**How users use environment?** + +Users can save different environment configs which can be also shared across the platform. Environment profiles can be used to run a notebook (e.g. by choosing different kernel from Jupyter), or an experiment. Predefined experiment library includes what environment to use so users don't have to choose which environment to use. + +``` + + +-------------------+ + |+-----------------+| +------------+ + || Python=3.7 || |User1 | + || Tensorflow=2.0 || +------------+ + |+---Kernel -------+| +------------+ + |+-----------------+|<----+ |User2 | + ||OS=Ubuntu16.04 || + +------------+ + ||CUDA=10.2 || | +------------+ + ||GPU_Driver=375.. || | |User3 | + |+---Base Library--+| | +------------+ + +-----Default-Env---+ | + | + | + +-------------------+ | + |+-----------------+| | + || Python=3.3 || | + || Tensorflow=2.0 || | + |+---kernel--------+| | + |+-----------------+| | + ||OS=Ubuntu16.04 || | + ||CUDA=10.3 ||<----+ + ||GPU_Driver=375.. || + |+---Base Library--+| + +-----My-Customized-+ +``` + +There're two environments in the above graph, "Default-Env" and "My-Customized", which can have different combinations of libraries for different experiments/notebooks. Users can choose different environments for different experiments as they want. + +Environments can be added/listed/deleted/selected through CLI/SDK/UI. + +# Implementation + +## Environment API definition + +Let look at what object definition looks like to define an environment, API of environment looks like: + +``` + name: "my_submarine_env", + vm-image: "...", + docker-image: "...", + kernel: + + description: "this is the most common env used by team ABC" +``` + +- `vm-image` is optional if we don't need to launch new VM (like running a training job in a cloud-remote machine). +- `docker-image` is required +- `kernel` could be optional if kernel is already included by vm-image or docker-image. +- `name` of the environment should be unique in the system, so user can reference it when create a new experiment/notebook. + +## VM-image and Docker-image + +Docker-image and VM image should be prepared by system admin / SREs, it is hard for Data-Scientists to write an error-proof Dockerfile, and push/manage Docker images. This is one of the reason we hide Docker-image inside "environment", we will encourage users to customize their kernels if needed, but don't have to touch Dockerfile and build/push/manage new Docker images. + +As a project, we will document what's the best practice and example of Dockerfiles. + +Dockerfile should include proper `ENTRYPOINT` definition which pointed to our default script, so no matter it is notebook, or an experiment, we will setup kernel (see below) and other environment variables properly. + +## Kernel Implementation + +After investigating different alternatives (such as pipenv, venv, etc.), we decided to use Conda environment which nicely replaces Python virtual env, pip, and can also support other languages. More details can be found at: https://medium.com/@krishnaregmi/pipenv-vs-virtualenv-vs-conda-environment-3dde3f6869ed + +When once Conda, users can easily add, remove dependency of a Conda environment. User can also easily export environment to yaml file. + +The yaml file of Conda environment by using `conda env export` looks like: + +``` +name: base +channels: + - defaults +dependencies: + - _ipyw_jlab_nb_ext_conf=0.1.0=py37_0 + - alabaster=0.7.12=py37_0 + - anaconda=2020.02=py37_0 + - anaconda-client=1.7.2=py37_0 + - anaconda-navigator=1.9.12=py37_0 + - anaconda-project=0.8.4=py_0 + - applaunchservices=0.2.1=py_0 +``` + +Including Conda kernel, the environment object may look like: + +``` +name: "my_submarine_env", + vm-image: "...", + docker-image: "...", + kernel: + name: team_default_python_3.7 + channels: + - defaults + dependencies: + - _ipyw_jlab_nb_ext_conf=0.1.0=py37_0 + - alabaster=0.7.12=py37_0 + - anaconda=2020.02=py37_0 + - anaconda-client=1.7.2=py37_0 + - anaconda-navigator=1.9.12=py37_0 +``` + +When launch a new experiment / notebook session using the `my_submarine_env`, submarine server will use defined Docker image, and Conda kernel to launch of container. + +## Storage of Environment + +Environment of Submarine is just a simple text file, so it will be persisted in Submarine metastore, which is ideally a Database. + +Docker image is stored inside a regular Docker registry, which will be handled outside of the system. + +Conda dependencies are stored in Conda channel (where referenced packages are stored), which will be handled/setuped separately. (Popular conda channels are `default` and `conda-forge`) + +For more detailed discussion about storage-related implementations, please refer to [storage-implementation](./storage-implementation,md). + +## How to implement to make user can easily use Submarine environments? + +We like simplicities, and we don't want to leak complexities of implementations to the users. To make it happen, we have to do some works to hide complexities. + +There're two primary uses of environments: experiments and notebook, for both of them, users should not do works like explictily call `conda active $env_name` to active environments. To make it happen, what we can do is to include following parts in Dockerfile + +``` +FROM ubuntu:18.04 + + + + + + +# This is just a sample of Dockerfile, users can do more customizations if needed +ENTRYPOINT ["/submarine-bootstrap.sh"] +``` + +When Submarine Server (this is implementation detail of Submarine Server, user will not see it at all) launch an experiment, or notebook, it will invoke following `docker run` command (or any other equvilant like using K8s spec): + +``` +docker run --kernel -- .... python train.py --batch_size 5 (and other parameters) +``` + +Similarily, to launch a notebook: + +``` +docker run --kernel -- .... jupyter +``` + +The `submarine-bootstrap.sh` is part of Submarine repo, and will handle `--kernel` argument which will invoke `conda active $kernel_name` before anything else. (Like run the training job). + + + diff --git a/docs/design/experiment-implementation.md b/docs/design/experiment-implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..ccc37a52c9114de1f15c4c585426f7c25e21c0d7 --- /dev/null +++ b/docs/design/experiment-implementation.md @@ -0,0 +1,500 @@ + + +# Experiment Implementations + +## Overview + +This document talks about implementation of experiment, flows and design considerations. + +Experiment consists of following components, also interact with other Submarine or 3rd-party components, showing below: + +``` + + + +---------------------------------------+ + +----------+ | Experiment Tasks | + |Run | | | + |Configs | | +----------------------------------+ | + +----------+ | | Experiment Runnable Code | | +-----------------+ + +----------+ | | | | |Output Artifacts | + |Input Data| | | (Like train-job.py) | | |(Models, etc.) | + | | | +----------------------------------+ | +-----------------+ + | | | +----------------------------------+ | + +----------+ | | Experiment Deps (Like Python) | | +-------------+ + | +----------------------------------+ | |Logs/Metrics | + | +----------------------------------+ | | | + | | OS, Base Libaries (Like CUDA) | | +-------------+ + | +----------------------------------+ | + +---------------------------------------+ + ^ + | (Launch Task with resources) + + + +---------------------------------+ + |Resource Manager (K8s/YARN/Cloud)| + +---------------------------------+ +``` + +As showing in the above diagram, Submarine experiment consists of the following items: + +- On the left side, there're input data and run configs. +- In the middle box, they're experiment tasks, it could be multiple tasks when we run distributed training, pipeline, etc. + - There're main runnable code, such as `train.py` for the training main entry point. + - The two boxes below: experiment dependencies and OS/Base libraries we called `Submarine Environment Profile` or `Environment` for short. Which defined what is the basic libraries to run the main experiment code. + - Experiment tasks are launched by Resource Manager, such as K8s/YARN/Cloud or just launched locally. There're resources constraints for each experiment tasks. (e.g. how much memory, cores, GPU, disk etc. can be used by tasks). +- On the right side, they're artifacts generated by experiments: + - Output artifacts: Which are main output of the experiment, it could be model(s), or output data when we do batch prediction. + - Logs/Metrics for further troubleshooting or understanding of experiment's quality. + +For the rest of the design doc, we will talk about how we handle environment, code, and manage output/logs, etc. + +## API of Experiment + +This is not a full definition of experiment, for more details, please reference to experiment API. + +Here's just an example of experiment object which help developer to understand what included in an experiment. + +```yaml +experiment: + name: "abc", + type: "script", + environment: "team-default-ml-env" + code: + sync_mode: s3 + url: "s3://bucket/training-job.tar.gz" + parameter: > python training.py --iteration 10 + --input=s3://bucket/input output=s3://bucket/output + resource_constraint: + res="mem=20gb, vcore=3, gpu=2" + timeout: "30 mins" +``` + +This defined a "script" experiment, which has a name "abc", the name can be used to track the experiment. There's environment "team-default-ml-env" defined to make sure dependencies of the job can be downloaded properly before executing the job. + +`code` defined where the experiment code will be downloaded, we will support a couple of sync_mode like s3 (or abfs/hdfs), git, etc. + +Different types of experiments will have different specs, for example distributed Tensorflow spec may look like: + +```yaml +experiment: + name: "abc-distributed-tf", + type: "distributed-tf", + ps: + environment: "team-default-ml-cpu" + resource_constraint: + res="mem=20gb, vcore=3, gpu=0" + worker: + environment: "team-default-ml-gpu" + resource_constraint: + res="mem=20gb, vcore=3, gpu=2" + code: + sync_mode: git + url: "https://foo.com/training-job.git" + parameter: > python /code/training-job/training.py --iteration 10 + --input=s3://bucket/input output=s3://bucket/output + tensorboard: enabled + timeout: "30 mins" +``` + +Since we have different Docker image, one is using GPU and one is not using GPU, we can specify different environment and resource constraint. + +## Manage environments for experiment + +Please refer to [environment-implementation.md](./environments-implementation.md) for more details + +## Manage storages for experiment + +There're different types of storage, such as logs, metrics, dependencies (environments). For more details. Please refer to [storage-implementations](./storage-implementation.md) for more details. This also includes how to manage code for experiment code. + +## Manage Pre-defined experiment libraries + +## Flow: Submit an experiment + +### Submit via SDK Flows. + +To better understand experiment implementation, It will be good to understand what is the steps of experiment submission. + +*Please note that below code is just pseudo code, not official APIs.* + +### Specify what environment to use + +Before submit the environment, you have to choose what environment to choose. Environment defines dependencies, etc. of an experiment or a notebook. might looks like below: + +``` +conda_environment = +""" + name: conda-env + channels: + - defaults + dependencies: + - asn1crypto=1.3.0=py37_0 + - blas=1.0=mkl + - ca-certificates=2020.1.1=0 + - certifi=2020.4.5.1=py37_0 + - cffi=1.14.0=py37hb5b8e2f_0 + - chardet=3.0.4=py37_1003 + prefix: /opt/anaconda3/envs/conda-env +""" + +# This environment can be different from notebook's own environment +environment = create_environment { + DockerImage = "ubuntu:16", + CondaEnvironment = conda_environment +} +``` + +To better understand how environment works, please refer to [environment-implementation](./environments-implementation.md). + +### Create experiment, specify where's training code located, and parameters. + +For ad-hoc experiment (code located at S3), assume training code is part of the `training-job.tar.gz` and main class is `train.py`. When the job is launched, whatever specified in the localize_artifacts will be downloaded. + +``` +experiment = create_experiment { + Environment = environment, + ExperimentConfig = { + type = "adhoc", + localize_artifacts = [ + "s3://bucket/training-job.tar.gz" + ], + name = "abc", + parameter = "python training.py --iteration 10 --input="s3://bucket/input output="s3://bucket/output", + } +} +experiment.run() +experiment.wait_for_finish(print_output=True) +``` + +##### Run notebook file in offline mode + +It is possible we want to run a notebook file in offline mode, to do that, here's code to use to run a notebook code + +``` +experiment = create_experiment { + Environment = environment, + ExperimentConfig = { + type = "adhoc", + localize_artifacts = [ + "s3://bucket/folder/notebook-123.ipynb" + ], + name = "abc", + parameter = "runipy training.ipynb --iteration 10 --input="s3://bucket/input output="s3://bucket/output", + } +} +experiment.run() +experiment.wait_for_finish(print_output=True) +``` + +##### Run pre-defined experiment library + +``` +experiment = create_experiment { + # Here you can use default environment of library + Environment = environment, + ExperimentConfig = { + type = "template", + name = "abc", + # A unique name of template + template = "deepfm_ctr", + # yaml file defined what is the parameters need to be specified. + parameter = { + Input: "S3://.../input", + Output: "S3://.../output" + Training: { + "batch_size": 512, + "l2_reg": 0.01, + ... + } + } + } +} +experiment.run() +experiment.wait_for_finish(print_output=True) +``` + +## Summarize: Experiment v.s. Notebook session + +There's a common misunderstanding about what is the differences between running experiment v.s. running task from a notebook session. We will talk about differences and commonalities: + +**Differences** + +| | Experiment | Notebook Session | +| --------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | +| Run mode | Offline | Interactive | +| Output Artifacts (a.k.a model) | Persisted in a shared storage (like S3/NFS) | Local in the notebook session container, could be ephemeral | +| Run history (meta, logs, metrics) | Meta/logs/metrics can be traced from experiment UI (or corresponding API) | No run history can be traced from Submarine UI/API. Can view the current running paragraph's log/metrics, etc. | +| What to run? | Code from Docker image or shared storage (like Tarball on S3, Github, etc.) | Local in the notebook's paragraph | + +**Commonalities** + +| | Experiment & Notebook Session | +| ----------- | ------------------------------------------------- | +| Environment | They can share the same Environment configuration | + +## Experiment-related modules inside Submarine-server + +(Please refer to [architecture of submarine server](./submarine-server/architecture.md) for more details) + +### Experiment Manager + +The experiment manager receives the experiment requests, persisting the experiment metas in a database(e.g. MySQL), will invoke subsequence modules to submit and monitor the experiment's execution. + +### Compute Cluster Manager + +After experiment accepted by experiment manager, based on which cluster the experiment intended to run (like mentioned in the previous sections, Submarine supports to manage multiple compute clusters), compute cluster manager will returns credentials to access the compute cluster. It will also be responsible to create a new compute cluster if needed. + +For most of the on-prem use cases, there's only one cluster involved, for such cases, ComputeClusterManager returns credentials to access local cluster if needed. + +### Experiment Submitter + +Experiment Submitter handles different kinds of experiments to run (e.g. ad-hoc script, distributed TF, MPI, pre-defined templates, Pipeline, AutoML, etc.). And such experiments can be managed by different resource management systems (e.g. K8s, YARN, container cloud, etc.) + +To meet the requirements to support variant kinds of experiments and resource managers, we choose to use plug-in modules to support different submitters (which requires jars to submarine-server’s classpath). + +To avoid jars and dependencies of plugins break the submarine-server, the plug-ins manager, or both. To solve this issue, we can instantiate submitter plug-ins using a classloader that is different from the system classloader. + +#### Submitter Plug-ins + +Each plug-in uses a separate module under the server-submitter module. As the default implements, we provide for YARN and K8s. For YARN cluster, we provide the submitter-yarn and submitter-yarnservice plug-ins. The submitter-yarn plug-in used the [TonY](https://github.com/linkedin/TonY) as the runtime to run the training job, and the submitter-yarnservice plug-in direct use the [YARN Service](https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/yarn-service/Overview.html) which supports Hadoop v3.1 above. + +The submitter-k8s plug-in is used to submit the job to Kubernetes cluster and use the [operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) as the runtime. The submitter-k8s plug-in implements the operation of CRD object and provides the java interface. In the beginning, we use the [tf-operator](https://github.com/kubeflow/tf-operator) for the TensorFlow. + +If Submarine want to support the other resource management system in the future, such as submarine-docker-cluster (submarine uses the Raft algorithm to create a docker cluster on the docker runtime environment on multiple servers, providing the most lightweight resource scheduling system for small-scale users). We should create a new plug-in module named submitter-docker under the server-submitter module. + +### Experiment Monitor + +The monitor tracks the experiment life cycle and records the main events and key info in runtime. As the experiment run progresses, the metrics are needed for evaluation of the ongoing success or failure of the execution progress. Due to adapt the different cluster resource management system, so we need a generic metric info structure and each submitter plug-in should inherit and complete it by itself. + +### Invoke flows of experiment-related components + +``` + +-----------------+ +----------------+ +----------------+ +-----------------+ + |Experiments | |Compute Cluster | |Experiment | | Experiment | + |Mgr | |Mgr | |Submitter | | Monitor | + +-----------------+ +----------------+ +----------------+ +-----------------+ + + + + + + User | | | | + Submit |+------------------------------------->+ + + Xperiment| Use submitter.validate(spec) | | + | to validate spec and create | | + | experiment object (state- | | + | machine). | | + | | | + | The experiment manager will | | + | persist meta-data to Database| | + | | | | + | | + + + |+-----------------> + | | + | Submit Experiments| | | + | To ComputeCluster| | | + | Mgr, get existing|+---------------->| | + | cluster, or | Use Submitter | | + | create a new one.| to submit |+---------------> | + | | Different kinds | Once job is | + | | of experiments | submitted, use |+----+ + | | to k8s/yarn, etc| monitor to get | | + | | | status updates | | + | | | | | Monitor + | | | | | Xperiment + | | | | | status + | | | | | + |<--------------------------------------------------------+| | + | | | | | + | Update Status back to Experiment | | + | | Manager | |<----+ + | | | | + | | | | + | | | | + v v v v +``` + +TODO: add more details about template, environment, etc. + +## Common modules of experiment/notebook-session/model-serving + +Experiment/notebook-session/model-serving share a lot of commonalities, all of them are: + +- Some workloads running on YARN/K8s. +- Need persist meta data to DB. +- Need monitor task/service running status from resource management system. + +We need to make their implementation are loose-coupled, but at the same time, share some building blocks as much as possible (e.g. submit PodSpecs to K8s, monitor status, get logs, etc.) to reduce duplications. + +## Support Predefined-experiment-templates + +Predefined Experiment Template is just a way to save data-scientists time to repeatedly entering parameters which is not error-proof and user experience is also bad. + +### Predefined-experiment-template API to run experiment + +Predefined experiment template consists a list of parameters, each of the parameter has 4 properties: + +| Key | Required | Default Value | Description | +| --------------- | ---------- | ------------------------------------------------------------ | ---------------------------- | +| Name of the key | true/false | When required = false, a default value can be provided by the template | Description of the parameter | + +For the example of deepfm CTR training experiment mentioned in the [architecture-and-requirements.md](./architecture-and-requirements.md) + +``` +{ + "input": { + "train_data": ["hdfs:///user/submarine/data/tr.libsvm"], + "valid_data": ["hdfs:///user/submarine/data/va.libsvm"], + "test_data": ["hdfs:///user/submarine/data/te.libsvm"], + "type": "libsvm" + }, + "output": { + "save_model_dir": "hdfs:///user/submarine/deepfm", + "metric": "auc" + }, + "training": { + "batch_size" : 512, + "field_size": 39, + "num_epochs": 3, + "feature_size": 117581, + ... + } +} +``` + +The template will be (in yaml format): + +```yaml +# deepfm.ctr template +name: deepfm.ctr +author: +description: > + This is a template to run CTR training using deepfm algorithm, by default it runs + single node TF job, you can also overwrite training parameters to use distributed + training. + +parameters: + - name: input.train_data + required: true + description: > + train data is expected in SVM format, and can be stored in HDFS/S3 + ... + - name: training.batch_size + required: false + default: 32 + description: This is batch size of training +``` + +The batch format can be used in UI/API. + +### Handle Predefined-experiment-template from server side + +Please note that, the conversion of predefined-experiment-template will be always handled by server. The invoke flow looks like: + +``` + + +------------Submarine Server -----------------------+ + +--------------+ | +-----------------+ | + |Client |+------->|Experimment Mgr | | + | | | | | | + +--------------+ | +-----------------+ | + | + | + Submit | +-------v---------+ Get Experiment Template | + Template | |Experiment |<-----+From pre-registered | + Parameters | |Template Registry| Templates | + to Submarine | +-------+---------+ | + Server | | | + | +-------v---------+ +-----------------+ | + | |Deepfm CTR Templ-| |Experiment- | | + | |ate Handler +------>|Tensorflow | | + | +-----------------+ +--------+--------+ | + | | | + | | | + | +--------v--------+ | + | |Experiment | | + | |Submitter | | + | +--------+--------+ | + | | | + | | | + | +--------v--------+ | + | | | | + | | ...... | | + | +-----------------+ | + | | + +----------------------------------------------------+ +``` + +Basically, from Client, it submitted template parameters to Submarine Server, inside submarine server, it finds the corresponding template handler based on the name. And the template handler converts input parameters to an actual experiment, such as a distributed TF experiment. After that, it goes the similar route to validate experiment spec, compute cluster manager, etc. to get the experiment submitted and monitored. + +Predefined-experiment-template is able to create any kind of experiment, it could be a pipeline: + +``` + + +-----------------+ +------------------+ + |Template XYZ | | XYZ Template | + | |+---------------> | Handler | + +-----------------+ +------------------+ + + + | + | + | + | + v + +--------------------+ +------------------+ + | +-----------------+| | Predefined | + | | Split Train/ ||<----+| Pipeline | + | | Test data || +------------------+ + | +-------+---------+| + | | | + | +-------v---------+| + | | Spark Job ETL || + | | || + | +-------+---------+| + | | | + | +-------v---------+| + | | Train using || + | | XGBoost || + | +-------+---------+| + | | | + | +-------v---------+| + | | Validate Train || + | | Results || + | +-----------------+| + | | + +--------------------+ +``` + +Template can be also chained to reuse other template handlers + +``` + + +-----------------+ +------------------+ + |Template XYZ | | XYZ Template | + | |+---------------> | Handler | + +-----------------+ +------------------+ + + + | + v + +------------------+ +------------------+ + |Distributed | | ABC Template | + |TF Experiment |<----+| Handler | + +------------------+ +------------------+ +``` + +Template Handler is a callable class inside Submarine Server with a standard interface defined like. + +```java +interface ExperimentTemplateHandler { + ExperimentSpec createExperiment(TemplatedExperimentParameters param) +} +``` + +We should avoid users to do coding when they want to add new template, we should have several standard template handler to deal with most of the template handling. + +Experiment templates can be registered/updated/deleted via Submarine Server's REST API, which need to be discussed separately in the doc. (TODO) diff --git a/docs/design/implementation-notes.md b/docs/design/implementation-notes.md new file mode 100644 index 0000000000000000000000000000000000000000..12417eb6e4cea74c0efa498d5826f071349e737e --- /dev/null +++ b/docs/design/implementation-notes.md @@ -0,0 +1,31 @@ + + +# Submarine Implementation Notes + +Before digging into details of implementations, you should read [architecture-and-requirements](./architecture-and-requirements.md) first to understand overall requirements and architecture. + +Here're sub topics of Submarine implementations: + +- [Submarine Storage](./storage-implementation.md): How to store metadata, logs, metrics, etc. of Submarine. +- [Submarine Environment](./environments-implementation.md): How environments created, managed, stored in Submarine. +- [Submarine Experiment](./experiment-implementation.md): How experiments managed, stored, and how the predefined experiment template works. +- [Submarine Notebook](./notebook-implementation.md): How experiments managed, stored, and how the predefined experiment template works. +- [Submarine Server](./submarine-server/architecture.md): How Submarine server is designed, architecture, implementation notes, etc. + +Working-in-progress designs, Below are designs which are working-in-progress, we will move them to the upper section once design & review is finished: + +- [Submarine HA Design](./wip-designs/SubmarineClusterServer.md): How Submarine HA can be achieved, using RAFT, etc. +- [Submarine services deployment module:](./wip-designs/submarine-launcher.md) How to deploy submarine services to k8s, YARN or cloud. \ No newline at end of file diff --git a/docs/design/notebook-implementation.md b/docs/design/notebook-implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..cac946b93d581a3348a94e568c464c3d8854f624 --- /dev/null +++ b/docs/design/notebook-implementation.md @@ -0,0 +1,111 @@ + + +# Submarine Notebook Implementation + +## Overview + +### User's interaction + +Users can start N (N >= 0) number of Notebook sessions, a notebook session is a running notebook instance. + +- Notebook session can be launched by Submarine UI (P0), and Submarine CLI (P2). +- When launch notebook session, users can choose T-shirt size of notebook session (how much mem/cpu/gpu resources, or resource profile such as small, medium, large, etc.). (P0) +- And user can choose an environment for notebook. More details please refer to [environment implementation](./environments-implementation.md) (P0) +- When start a notebook, user can choose what code to be initialized, similar to experiment. (P1) +- Optionally, users can choose to attach a persistent volume to a notebook session. (P2) + +Users can get a list of notebook sessions belongs to themselves, and connect to notebook session. + +User can choose to terminate a running notebook session. + +### Admin's interaction + +- How many concurrent notebook sessions can be launched by each user is determined by resource quota limits of each user, and maximum concurrent notebook sessions can be launched by each user. (P2) + +## Relationship with other components + +### Metadata store + +Running notebook sessions' metadata need persistented in Submarine's metadata store (Database). + +### Submarine Server + +``` + + +--------------+ +--------Submarine Server--------------------+ + |Submarine UI | | +-------------------+ | + | |+---> Submarine | | + | Notebook | | | Notebook REST API| | + +--------------+ | | | | + | +--------+----------+ +--------------+ | + | | +->|Metastore | | + | +--------v----------+ | |DB | | + | | Submarine +--+ +--------------+ | + | | Notebook Mgr | | + | | | | + | | | | + | +--------+----------+ | + | | | + +----------|---------------------------------+ + | + +--------------+ + +--------v---------+ + | Notebook Session | + | | + | instance | + | | + +------------------+ +``` + +Once user use Submarine UI to launch a notebook session, Submarine notebook manager inside Submarine Server will persistent notebook session's metadata, and launch a new notebook session instance. + +### Resource manager + +When using K8s as resource manager, Submarine notebook session will run as a new POD. + +### Storage + +There're several different types of storage requirements for Submarine notebook. + +For code, environment, etc, storage, please refer to [storage implementation](./storage-implementation.md), check "Localization of experiment/notebook/model-serving code". + +When there're needs to attach volume (such as user's home folder) to Submarine notebook session, please check [storage implementation](./storage-implementation.md), check "Attachable volume". + +### Environment + +Submarine notebook's environment should be used to run experiment, model serving, etc. Please check [environment implementation](./environments-implementation.md). (More specific to notebook, please check "How to implement to make user can easily use Submarine environments") + +Please note that notebook's Environment should include right version of notebook libraries, and admin should follow the guidance to build correct Docker image, Conda libraries to correctly run Notebook. + +### Submarine SDK (For Experiment, etc.) + +Users can run new experiment, access metrics information, or do model operations using Submarine SDK. + +Submarine SDK is a Python library which can talk to Submarine Server which need Submarine Server's endpoint as well as user credentials. + +To ensure better experience, we recommend always install proper version of Submarine SDK from environment which users can use Submarine SDK directly from commandline. (We as Submarine community can provide sample Dockerfile or Conda environment which have correct base libraries installed for Submarine SDK). + +Submarine Server IP will be configured automatically by Submarine Server, and added as an envar when Submarine notebook session got launched. + +### Security + +Please refer to [Security Implementation](./security-implementation.md) + +Once user accessed to a running notebook session, the user can also access resources of the notebook, capability of submit new experiment, and access data. This is also very dangerous so we have to protect it. + +A simple solution is to use token-based authentication https://jupyter-notebook.readthedocs.io/en/stable/security.html. A more common way is to use solutions like KNOX to support SSO. + +We need expand this section to more details. (TODO). \ No newline at end of file diff --git a/docs/design/security-implementation.md b/docs/design/security-implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..611f5bcc15845d89de617024a4235b846b1016b6 --- /dev/null +++ b/docs/design/security-implementation.md @@ -0,0 +1,24 @@ + + +# Submarine Security Implementation + +## Handle User's Credential + +Users credential includes Kerberoes Keytabs, Docker registry credentials, Github ssh-keys, etc. + +User's credential must be stored securitely, for example, via KeyCloak or K8s Secrets. + +(More details TODO) \ No newline at end of file diff --git a/docs/design/storage-implementation.md b/docs/design/storage-implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..8b2961d988d782397c32a25ce2404837ab9911e7 --- /dev/null +++ b/docs/design/storage-implementation.md @@ -0,0 +1,165 @@ + + +# Submarine Storage Implementation + +## ML-related objects and their storages + +First let's look at what user will interact for most of the time: + +- Notebook +- Experiment +- Model Servings + +``` + + + +---------+ +------------+ + |Logs |<--+|Notebook | + +----------+ +---------+ +------------+ +----------------+ + |Trackings | <-+|Experiment |<--+>|Model Artifacts | + +----------+ +-----------------+ +------------+ +----------------+ + +----------+<---+|ML-related Metric|<--+Servings | + |tf.events | +-----------------+ +------------+ + +----------+ ^ +-----------------+ + + | Environments | + +----------------------+ | | + +-----------------+ | Submarine Metastore | | Dependencies | + |Code | +----------------------+ | | + +-----------------+ |Experiment Meta | | Docker Images | + +----------------------+ +-----------------+ + |Model Store Meta | + +----------------------+ + |Model Serving Meta | + +----------------------+ + |Notebook meta | + +----------------------+ + |Experiment Templates | + +----------------------+ + |Environments Meta | + +----------------------+ +``` + +First of all, all the notebook-sessions / experiments / model-serving instances) are more or less interact with following storage objects: + +- Logs for these tasks for troubleshooting. +- ML-related metrics such as loss, epoch, etc. (in contrast of system metrics such as CPU/memory usage, etc.) + - There're different types of ML-related metrics, for Tensorflow/pytorch, they can use tf.events and get visualizations on tensorboard. + - Or they can use tracking APIs (such as Submarine tracking, mlflow tracking, etc.) to output customized tracking results for non TF/Pytorch workloads. +- Training jobs of experiment typically generate model artifacts (files) which need persisted, and both of notebook, model serving needs to load model artifacts from persistent storage. +- There're various of meta information, such as experiment meta, model registry, model serving, notebook, experiment, environment, etc. We need be able to read these meta information back. +- We also have code for experiment (like training/batch-prediction), notebook (ipynb), and model servings. +- And notebook/experiments/model-serving need depend on environments (dependencies such as pip, and Docker Images). + +### Implementation considerations for ML-related objects + +| Object Type | Characteristics | Where to store | +| ---------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | +| Metrics: tf.events | Time series data with k/v, appendable to file | Local/EBS, HDFS, Cloud Blob Storage | +| Metrics: other tracking metrics | Time series data with k/v, appendable to file | Local, HDFS, Cloud Blob Storage, Database | +| Logs | Large volumes, #files are potentially huge. | Local (temporary), HDFS (need aggregation), Cloud Blob Storage | +| Submarine Metastore | CRUD operations for small meta data. | Database | +| Model Artifacts | Size varies for model (from KBs to GBs). #files are potentially huge. | HDFS, Cloud Blob Storage | +| Code | Need version control. (Please find detailed discussions below for code storage and localization) | Tarball on HDFS/Cloud Blog Storage, or Git | +| Environment (Dependencies, Docker Image) | | Public/private environment repo (like Conda channel), Docker registry. | + +### Detailed discussions + +#### Store code for experiment/notebook/model-serving + +There're following ways to get experiment code: + +**1) Code is part of Git repo:** (***Recommended***) + +This is our recommended approach, once code is part of Git, it will be stored in version control, any change will be tracked, and much easier for users to trace back what change triggered a new bug, etc. + +**2) Code is part of Docker image:** + +***This is an anti-pattern and we will NOT recommend you to use it***, Docker image can be used to include ANYTHING, like dependencies, the code you will execute, or even data. But this doesn't mean you should do it. We recommend to use Docker image ONLY for libraries/dependencies. + +Making code to be part of Docker image makes hard to edit code (if you want to update a value in your Python file, you will have to recreate the Docker image, push it and rerun it). + +**3) Code is part of S3/HDFS/ABFS:** + +User may want to store their training code to a tarball on a shared storage. Submarine need to download code from remote storage to the launched container before running the code. + +#### Localization of experiment/notebook/model-serving code + +To make user experiences keeps same across different environment, we will localize code to a same folder after the container is launched, preferably `/code` + +For example, there's a git repo need to be synced up for an experiment/notebook/model-serving (example above): + +``` +experiment: #Or notebook, model-serving + name: "abc", + environment: "team-default-ml-env" + ... (other fields) + code: + sync_mode: git + url: "https://foo.com/training-job.git" +``` + +After localize, `training-job/` will be placed under `/code` + +When we running on K8s environment, we can use K8s's initContainer and emptyDir to do these things for us. K8s POD spec (generated by Submarine server instead of user, user should NEVER edit K8s spec, that's too unfriendly to data-scientists): + +``` +apiVersion: v1 +kind: Pod +metadata: + name: experiment-abc +spec: + containers: + - name: experiment-task + image: training-job + volumeMounts: + - name: code-dir + mountPath: /code + initContainers: + - name: git-localize + image: git-sync + command: "git clone .. /code/" + volumeMounts: + - name: code-dir + mountPath: /code + volumes: + - name: code-dir + emptyDir: {} +``` + +The above K8s spec create a code-dir and mount it to `/code` to launched containers. The initContainer `git-localize` uses `https://github.com/kubernetes/git-sync` to do the sync up. (If other storages are used such as s3, we can use similar initContainer approach to download contents) + +## System-related metrics/logs and their storages + +Other than ML-related objects, we have system-related objects, including: + +- Daemon logs (like logs of Submarine server). +- Logs for other dependency components (like Kubernetes logs when running on K8s). +- System metrics (Physical resource usages by daemons, launched training containers, etc.). + +All these information should be handled by 3rd party system, such as Grafana, Prometheus, etc. And system admins are responsible to setup these infrastructures, dashboard. Users of submarine should NOT interact with system related metrics/logs. It is system admin's responsibility. + +## Attachable Volumes + +It is possible user has needs to have an attachable volume for their experiment / notebook, this is especially useful for notebook storage, since contents of notebook can be automatically saved, and it can be used as user's home folder. + +Downside of attachable volume is, it is not versioned, even notebook is mainly used for adhoc exploring tasks, an unversioned notebook file can lead to maintenance issues in the future. + +Since this is a common requirement, we can consider to support attachable volumes in Submarine in a long run, but with relatively lower priority. + +## In-scope / Out-of-scope + + Describe what Submarine project should own and what Submarine project should NOT own. + diff --git a/docs/design/submarine-server/architecture.md b/docs/design/submarine-server/architecture.md new file mode 100644 index 0000000000000000000000000000000000000000..530dc0018e9e58bbea06d9001ee3cd493b6dc878 --- /dev/null +++ b/docs/design/submarine-server/architecture.md @@ -0,0 +1,165 @@ + +# Submarine Server Architecture And Implementation + +## Architecture Overview + +``` + +---------------Submarine Server ---+ + | | + | +------------+ +------------+ | + | |Web Svc/Prxy| |Backend Svc | | +--Submarine Asset + + | +------------+ +------------+ | |Project/Notebook | + | ^ ^ | |Model/Metrics | + +---|---------|---------------------+ |Libraries/Dataset | + | | +------------------+ + | | + | +--|-Compute Cluster 1---+ +--Image Registry--+ + + | | | | User's Images | + User / | + | | | + Admin | User Notebook Instance | +------------------+ + | Experiment Runs | + +------------------------+ +-Data Storage-----+ + | S3/HDFS, etc. | + +----Compute Cluster 2---+ | | + +------------------+ + ... +``` + +Here's a diagram to illustrate the Submarine's deployment. + +- Submarine Server consists of web service/proxy, and backend services. They're like "control planes" of Submarine, and users will interact with these services. +- Submarine server could be a microservice architecture and can be deployed to one of the compute clusters. (see below, this will be useful when we only have one cluster). +- There're multiple compute clusters that could be used by Submarine service. For user's running notebook instance, jobs, etc. they will be placed to one of the compute clusters by user's preference or defined policies. +- Submarine's asset includes project/notebook(content)/models/metrics/dataset-meta, etc. can be stored inside Submarine's own database. +- Datasets can be stored in various locations such as S3/HDFS. +- Users can push container (such as Docker) images to a preconfigured registry in Submarine, so Submarine service can know how to pull required container images. +- Image Registry/Data-Storage, etc. are outside of Submarine server's scope and should be managed by 3rd party applications. + +## Submarine Server and its APIs + +Submarine server is designed to allow data scientists to access notebooks, submit/manage jobs, manage models, create model training workflows, access datasets, etc. + +Submarine Server exposed UI and REST API. Users can also use CLI / SDK to manage assets inside Submarine Server. + +``` + +----------+ + | CLI |+---+ + +----------+ v +----------------+ + +--------------+ | Submarine | + +----------+ | REST API | | | + | SDK |+>| |+> Server | + +----------+ +--------------+ | | + ^ +----------------+ + +----------+ | + | UI |+---+ + +----------+ +``` + +REST API will be used by the other 3 approaches. (CLI/SDK/UI) + +The REST API Service handles HTTP requests and is responsible for authentication. It acts as the caller for the JobManager component. + +The REST component defines the generic job spec which describes the detailed info about job. For more details, refer to [here](https://docs.google.com/document/d/1kd-5UzsHft6gV7EuZiPXeWIKJtPqVwkNlqMvy0P_pAw/edit#). (Please note that we're converting REST endpoint description from Java-based REST API to swagger definition, once that is done, we should replace the link with swagger definition spec). + +## Proposal + ``` + +---------------------+ + +-----------+ | +--------+ +----+ | + | | | |runtime1+-->+job1| | + | workbench +---+ +----------------------------------+ | +--------+ +----+ | + | | | | +------+ +---------------------+ | +-->+ +--------+ +----+ | + +-----------+ | | | | | +------+ +-------+ | | | | |runtime2+-->+job2| | + | | | | | | YARN | | K8s | | | | | +--------+ +----+ | + +-----------+ | | | | | +------+ +-------+ | | | | YARN Cluster | + | | | | | | | submitter | | | +---------------------+ + | CLI +------>+ | REST | +---------------------+ +---+ + | | | | | | +---------------------+ | | +---------------------+ + +-----------+ | | | | | +-------+ +-------+ | | | | +--------+ +----+ | + | | | | | |PlugMgr| |monitor| | | | | | +-->+job1| | + +-----------+ | | | | | +-------+ +-------+ | | | | | | +----+ | + | | | | | | | JobManager | | +-->+ |operator| +----+ | + | SDK +---+ | +------+ +---------------------+ | | | +-->+job2| | + | | +----------------------------------+ | +--------+ +----+ | + +-----------+ | K8s Cluster | + client server +---------------------+ + ``` +We propose to split the original core module in the old layout into two modules, CLI and server as shown in FIG. The submarine-client calls the REST APIs to submit and retrieve the job info. The submarine-server provides the REST service, job management, submitting the job to cluster, and running job in different clusters through the corresponding runtime. + +## Submarine Server Components + +``` + + +----------------------Submarine Server--------------------------------+ + | +-----------------+ +------------------+ +--------------------+ | + | | Experiment | |Notebook Session | |Environment Mgr | | + | | Mgr | |Mgr | | | | + | +-----------------+ +------------------+ +--------------------+ | + | | + | +-----------------+ +------------------+ +--------------------+ | + | | Model Registry | |Model Serving Mgr | |Compute Cluster Mgr | | + | | | | | | | | + | +-----------------+ +------------------+ +--------------------+ | + | | + | +-----------------+ +------------------+ +--------------------+ | + | | DataSet Mgr | |User/Team | |Metadata Mgr | | + | | | |Permission Mgr | | | | + | +-----------------+ +------------------+ +--------------------+ | + +----------------------------------------------------------------------+ +``` + +### Experiment Manager + +TODO + +### Notebook Sessions Manager + +TODO + +### Environment Manager + +TODO + +### Model Registry + +TODO + +### Model Serving Manager + +TODO + +### Compute Cluster Manager + +TODO + +### Dataset Manager + +TODO + +### User/team permissions manager + +TODO + +### Metadata Manager + +TODO + +## Components/services outside of Submarine Server's scope + +TODO: Describe what are the out-of-scope components, which should be handled and managed outside of Submarine server. Candidates are: Identity management, data storage, metastore storage, etc. \ No newline at end of file diff --git a/docs/design/submarine-server/jobspec.md b/docs/design/submarine-server/jobspec.md new file mode 100644 index 0000000000000000000000000000000000000000..beb776a135019f52a9e7ba9b5450c3838c211b28 --- /dev/null +++ b/docs/design/submarine-server/jobspec.md @@ -0,0 +1,97 @@ + + +# Generic Job Spec + +## Motivation +As the machine learning platform, the submarine should support multiple machine learning frameworks, such as Tensorflow, Pytorch etc. But different framework has different distributed components for the training job. So that we designed a generic job spec to abstract the training job across different frameworks. In this way, the submarine-server can hide the complexity of underlying infrastructure differences and provide a cleaner interface to manager jobs + +## Proposal +Considering the Tensorflow and Pytorch framework, we propose one spec which consists of library spec, submitter spec and task specs etc. Such as: +```yaml +name: "mnist" +librarySpec: + name: "TensorFlow" + version: "2.1.0" + image: "apache/submarine:tf-mnist-with-summaries-1.0" + cmd: "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150" + envVars: + ENV_1: "ENV1" +submitterSpec: + type: "k8s" + namespace: "submarine" +taskSpecs: + Ps: + name: tensorflow + replicas: 2 + resources: "cpu=4,memory=2048M,nvidia.com/gpu=1" + Worker: + name: tensorflow + replicas: 2 + resources: "cpu=4,memory=2048M,nvidia.com/gpu=1" +``` + +### Library Spec +The library spec describes the info about machine learning framework. All the fields as below: + +| field | type | optional | description | +|---|---|---|---| +| name | string | NO | Machine Learning Framework name. Only `"tensorflow"` and `"pytorch"` is supported. It doesn't matter if the value is uppercase or lowercase.| +| version | string | NO | The version of ML framework. Such as: 2.1.0 | +| image | string | NO | The public image used for each task if not specified. Such as: apache/submarine | +| cmd | string | YES | The public entry cmd for the task if not specified. | +| envVars | key/value | YES | The public env vars for the task if not specified. | + +### Submitter Spec +It describes the info of submitter which the user specified, such as yarn, yarnservice or k8s. All the fields as below: + +| field | type | optional | description | +|---|---|---|---| +| type | string | NO | The submitter type, supports `k8s` now | +| configPath | string | YES | The config path of the specified resource manager. You can set it in submarine-site.xml if run submarine-server locally | +| namespace | string | NO | It's known as queue in Apache Hadoop YARN and namespace in Kubernetes. | +| kind | string | YES | It's used for k8s submitter, supports TFJob and PyTorchJob | +| apiVersion | string | YES | It should pair with the kind, such as the TFJob's api version is `kubeflow.org/v1` | + +### Task Spec +It describes the task info, the tasks make up the job. So it must be specified when submit the job. All the tasks should putted into the key value collection. Such as: +```yaml +taskSpecs: + Ps: + name: tensorflow + replicas: 2 + resources: "cpu=4,memory=2048M,nvidia.com/gpu=1" + Worker: + name: tensorflow + replicas: 2 + resources: "cpu=4,memory=2048M,nvidia.com/gpu=1" +``` + +All the fields as below: + +| field | type | optional | description | +|---|---|---|---| +| name | string | YES | The job task name, if not specify using the library name | +| image | string | YES | The job task image | +| cmd | string | YES | The entry command for running task | +| envVars | key/value | YES | The env vars for the task | +| resources | string | NO | The limit resource for the task. Formatter: cpu=%s,memory=%s,nvidia.com/gpu=%s | + +## Implements +For more info see [SUBMARINE-321](https://issues.apache.org/jira/browse/SUBMARINE-321) diff --git a/docs/design/wip-designs/SubmarineClusterServer.md b/docs/design/wip-designs/SubmarineClusterServer.md new file mode 100644 index 0000000000000000000000000000000000000000..8588018cb8faa80eb5565d5b7b39c96ed1e8c5dc --- /dev/null +++ b/docs/design/wip-designs/SubmarineClusterServer.md @@ -0,0 +1,162 @@ +# Submarine Cluster Server Design - High-Availability (working-in-progress) + +## Please note that this design doc is working-in-progress and need more works to complete. + + + +## Below is existing proposal: + +## Introduction +The Submarine system contains a total of two daemon services, Submarine Server and Workbench Server. + +Submarine Server mainly provides job submission, job scheduling, job status monitoring, and model online service for Submarine. + +Workbench Server is mainly for algorithm users to provide algorithm development, Python/Spark interpreter operation, and other services through Notebook. + +The goal of the Submarine project is to provide high availability and high-reliability services for big data processing, +algorithm development, job scheduling, model online services, model batch, and incremental updates. + +In addition to the high availability of big data and machine learning frameworks, +the high availability of Submarine Server and Workbench Server itself is a key consideration. + +## Requirement + +### Cluster Metadata Center + +Multiple Submarine (or Workbench) Server processes create a Submarine Cluster through the RAFT algorithm library. + +The cluster internally maintains a metadata center. All servers can operate the metadata. + +The RAFT algorithm ensures that multiple processes are simultaneously co-located. + +A data modification will not cause problems such as mutual coverage and dirty data. + +This metadata center stores data by means of key-value pairs. it can store/support a variety of data, +but it should be noted that metadata is only suitable for storing small amounts of data and cannot be used to replace data storage. + +### Service discovery + +By storing the information of the service or process in the metadata center, we can easily find the information of the service or process we need in any place, +for example, the IP address and port where the Python interpreter will be the process. Information is stored in metadata, +and other services can easily find process information through process IDs and connect to provide service discovery capabilities. + +### Cluster event + +In the entire Submarine cluster, the servers can communicate with each other and other child processes to send cluster events to each other. + +The service or process processes the corresponding programs according to the cluster events. For example, +the Workbench Server can be managed to Python. The interpreter process sends a shutdown event that controls the operation of the services and individual subprocesses throughout the cluster. + +Cluster events support both broadcast and separate delivery capabilities. + +### Independence + +We implement Submarine's clustering capabilities through the RAFT algorithm library, without relying on any external services (e.g. Zookeeper, Etcd, etc.) + +### Disadvantages + +Because the RAFT algorithm requires more than half of the servers available to ensure the normality of the RAFT algorithm, +if we need to turn on the clustering capabilities of Submarine (Workbench) Server, when more than half of the servers are unavailable, +some programs may appear abnormal. Of course, we also detected this in the system, downgrading the system or refusing to provide service status. + +## System design + +### Universal design + +Modular design, Submarine (Workbench) Server exists in the Submarine system, these two services need to provide clustering capabilities, +so we abstract the cluster function into a separate module for development so that Submarine (Workbench) Server can reuse the cluster function module. + +### ClusterConfigure + +Add a `submarine.server.addr` and `workbench.server.addr` configuration items in `submarine-site.xml`, `submarine.server.addr=ip1, ip2, ip3`, +through the IP list, the RAFT algorithm module in the server process can Cluster with other server processes. + +### ClusterServer + ++ The ClusterServer module encapsulates the RAFT algorithm module, which can create a service cluster and read and write metadata based on the two configuration items submarine.server.addr or workbench.server.addr. + ++ The cluster management service runs in each submarine server; + ++ The cluster management service establishes a cluster by using the atomix RaftServer class of the Raft algorithm library, maintains the ClusterStateMachine, +and manages the service state metadata of each submarine server through the PutCommand, GetQuery, and DeleteCommand operation commands. + +### ClusterClient + ++ The ClusterClient module encapsulates the RAFT algorithm client module, which can communicate with the cluster according to the two configuration items `submarine.server.addr` or `workbench.server.addr`, +read and write metadata, and write the IP and port information of the client process. Into the cluster's metadata center. + ++ The cluster management client runs in each submarine server and submarine Interpreter process; + ++ The cluster management client manages the submarine server and submarine Interpreter process state (metadata information) +in the ClusterStateMachine by using the atomix RaftClient class of the Raft library to connect to the atomix RaftServer. + ++ When the submarine server and Submarine Interpreter processes are started, they are added to the ClusterStateMachine and are removed from the ClusterStateMachine + ++ when the Submarine Server and Submarine Interpreter processes are closed. + +### ClusterMetadata +Metadata stores metadata information in a KV key-value pair。 +ServerMeta：key='host:port'，value= {SERVER_HOST=...，SERVER_PORT=...，...} + + + +| Name | Description | +| --------------------- | ------------------------------- | +| SUBMARINE_SERVER_HOST | Submarine server IP | +| SUBMARINE_SERVER_PORT | Submarine server port | +| WORKBENCH_SERVER_HOST | Submarine workbench server IP | +| WORKBENCH_SERVER_PORT | Submarine workbench server port | + +InterpreterMeta：key=InterpreterGroupId，value={INTP_TSERVER_HOST=...，...} + +| Name | Description | +| ----------------- | ------------------------------------ | +| INTP_TSERVER_HOST | Submarine Interpreter Thrift IP | +| INTP_TSERVER_PORT | Submarine Interpreter Thrift port | +| INTP_START_TIME | Submarine Interpreter start time | +| HEARTBEAT | Submarine Interpreter heartbeat time | + +### Network fault tolerance + +In a distributed environment, there may be network anomalies, network delays, or service exceptions. After submitting metadata to the cluster, +check whether the submission is successful. After the submission fails, save the metadata in the local message queue. A separate commit thread to retry; + +### Cluster monitoring + +The cluster needs to monitor whether the Submarine Server and Submarine-Interpreter processes are working properly. + +The Submarine Server and Submarine Interpreter processes periodically send heartbeats to update their own timestamps in the cluster metadata. + +The Submarine Server with Leader identity periodically checks the timestamps of the Submarine Server and Submarine Interpreter processes to clear the timeout services and processes. + +1. The cluster monitoring module runs in each Submarine Server and Submarine Interpreter process, +periodically sending heartbeat data of the service or process to the cluster; + +2. When the cluster monitoring module runs in Submarine Server, it sends the heartbeat to the cluster's ClusterStateMachine. +If the cluster does not receive heartbeat information for a long time, Indicates that the service or process is abnormal and unavailable. + +3. Resource usage statistics strategy, in order to avoid the instantaneous high peak and low peak of the server, +the cluster monitoring will collect the average resource usage in the most recent period for reporting, and improve the reasonable line and effectiveness of the server resources as much as possible; + +4. When the cluster monitoring module runs in the Submarine Server, it checks the heartbeat data of each Submarine Server and Submarine Interpreter process. +If it times out, it considers that the service or process is abnormally unavailable and removes it from the cluster. + +### Atomix Raft algorithm library + +In order to reduce the deployment complexity of distributed mode, submarine server does not use Zookeeper to build a distributed cluster. +Multiple submarine server groups are built into distributed clusters by using the Raft algorithm in submarine server. +The Raft algorithm is involved by atomix lib of atomix that has passed Jepsen consistency verification. + +### Synchronize workbench notes + +In cluster mode, the user creates, modifies, and deletes the note on any of the servers. +All need to be notified to all the servers in the cluster to synchronize the update of Notebook. +Failure to do so will result in the user not being able to continue while switching to another server. + +### Listen for note update events + +Listen for the NEW_NOTE, DEL_NOTE, REMOVE_NOTE_TO_TRASH ... event of the notebook in the NotebookServer#onMessage() function. + +### Broadcast note update event + +The note is refreshed by notifying the event to all Submarine servers in the cluster via messaging Service. diff --git a/docs/design/wip-designs/submarine-launcher.md b/docs/design/wip-designs/submarine-launcher.md new file mode 100644 index 0000000000000000000000000000000000000000..7465267a429a9fbfba648860c4df5bb5f6d4c026 --- /dev/null +++ b/docs/design/wip-designs/submarine-launcher.md @@ -0,0 +1,75 @@ +# Submarine Launcher + +## Introduction +Submarine is built and run in Cloud Native, taking advantage of the cloud computing model. + +To give full play to the advantages of cloud computing. +These applications are characterized by rapid and frequent build, release, and deployment. +Combined with the features of cloud computing, they are decoupled from the underlying hardware and operating system, +and can easily meet the requirements of scalability, availability, and portability. And provide better economy. + +In the enterprise data center, submarine can support k8s/yarn/docker three resource scheduling systems; +in the public cloud environment, submarine can support these cloud services in GCE/AWS/Azure; + +## Requirement + +### Cloud-Native Service + +The submarine server is a long-running services in the daemon mode. +The submarine server is mainly used by algorithm engineers to provide online front-end functions such as algorithm development, +algorithm debugging, data processing, and workflow scheduling. +And submarine server also mainly used for back-end functions such as scheduling and execution of jobs, tracking of job status, and so on. + +Through the ability of rolling upgrades, we can better provide system stability. +For example, we can upgrade or restart the workbench server without affecting the normal operation of submitted jobs. + +You can also make full use of system resources. +For example, when the number of current developers or job tasks increases, +The number of submarine server instances can be adjusted dynamically. + +In addition, submarine will provide each user with a completely independent workspace container. +This workspace container has already deployed the development tools and library files commonly used by algorithm engineers including their operating environment. +Algorithm engineers can work in our prepared workspaces without any extra work. + +Each user's workspace can also be run through a cloud service. + +### Service discovery +With the cluster function of submarine, each service only needs to run in the container, +and it will automatically register the service in the submarine cluster center. +Submarine cluster management will automatically maintain the relationship between service and service, service and user. + +## Design + +![cloud-service](../assets/design/multi-dc-cloud.png) + + +### Launcher + +The submarine launcher module defines the complete interface. +By using this interface, you can run the submarine server, and workspace in k8s / yarn / docker / AWS / GCE / Azure. + + +### Launcher On Docker +In order to allow some small and medium-sized users without k8s/yarn to use submarine, +we support running the submarine system in docker mode. + +Users only need to provide several servers with docker runtime environment. +The submarine system can automatically cluster these servers into clusters, manage all the hardware resources of the cluster, +and run the service or workspace container in this cluster through scheduling algorithms. + + +### Launcher On Kubernetes + +submarine operator + +### Launcher On Yarn +[TODO] + +### Launcher On AWS +[TODO] + +### Launcher On GCP +[TODO] + +### Launcher On Azure +[TODO] diff --git a/docs/development-guide-home.md b/docs/development-guide-home.md new file mode 100644 index 0000000000000000000000000000000000000000..6035df725e52ed6709c626eb684cb0c9904d60bb --- /dev/null +++ b/docs/development-guide-home.md @@ -0,0 +1,30 @@ + + +# Development Guide + +## Working-in-progress components + +### Submarine Workbench + +[Submarine Workbench Guide](./workbench/README.md) + +## Development + +[Submarine Project Development Guide](./development/README.md) + +[Submarine Project Database Guide](./database/README.md) + +[Submarine Quick Development Tips](./development/QuickDevTips.md) \ No newline at end of file diff --git a/docs/development/BuildFromCode.md b/docs/development/BuildFromCode.md new file mode 100644 index 0000000000000000000000000000000000000000..5ede0c96536410e44e95e2f22af71c7b32aff39c --- /dev/null +++ b/docs/development/BuildFromCode.md @@ -0,0 +1,96 @@ + + +## Build Submarine From Source Code + +## Prerequisites + ++ JDK 1.8 ++ Maven 3.3 or later ( 3.6.2 is known to fail, see SUBMARINE-273 ) ++ Docker + +## Quick Start + +### Build Your Custom Submarine Docker Images + +Submarine provides default Docker image in the release artifacts, sometimes you would like to do some modifications on the images. You can rebuild Docker image after you make changes. + +> Note that you need to make sure the images built above can be accessed in k8s +> Usually this needs a rename and push to a proper Docker registry. + +```bash +mvn clean package -DskipTests +``` + +Build submarine server image: + +```bash +./dev-support/docker-images/submarine/build.sh +``` + +Build submarine database image: + +```bash +./dev-support/docker-images/database/build.sh +``` + +### Building source code / binary distribution + ++ Checking releases for licenses + +``` +mvn clean org.apache.rat:apache-rat-plugin:check +``` + ++ Create binary distribution with default hadoop version + +``` +mvn clean package -DskipTests +``` + ++ Create binary distribution with hadoop-2.9.x version + +``` +mvn clean package -DskipTests -Phadoop-2.9 +``` + ++ Create binary distribution with hadoop-2.10.x version + +``` +mvn clean package -DskipTests -Phadoop-2.10 +``` + ++ Create binary distribution with hadoop-3.1.x version + +``` +mvn clean package -DskipTests -Phadoop-3.1 +``` + ++ Create binary distribution with hadoop-3.2.x version + +``` +mvn clean package -DskipTests -Phadoop-3.2 +``` + ++ Create source code distribution + +``` +mvn clean package -DskipTests -Psrc +``` + +## TonY code modification + +If it is needed to make modifications to TonY project, please make a PR +to [Tony repository](https://github.com/linkedin/TonY). \ No newline at end of file diff --git a/docs/development/IntegrationTest.md b/docs/development/IntegrationTest.md new file mode 100644 index 0000000000000000000000000000000000000000..3acb7aedfcf7ff32c2af0b56ecdd5358c946fb02 --- /dev/null +++ b/docs/development/IntegrationTest.md @@ -0,0 +1,122 @@ + + +# Integration Test +## Introduction +1. Now, Apache Submarine supports two kinds of integration test: `test-e2e` and `test-k8s`. These two modules can be found in the [submarine/submarine-test](https://github.com/apache/submarine/tree/master/submarine-test) directory. + +2. Currently, there are some differences between `test-e2e` and `test-k8s` in operation mode. To elaborate, `test-e2e` needs to deploy Apache Submarine locally, while `test-k8s` deploys Apache Submarine via k8s. + +3. These two test modules can be applied to different test scenarios. (In the future, these two test modules may be combined or adjusted) + +## k8s test + +k8s test: When the user submits the code to his/her repository or the `apache/submarine` git repository, the travis test task will automatically start. + +test-k8s runs test cases in travis. It will first create a k8s cluster by using the kind tool in travis, + +and then compile and package the submarine project in `submarine-dist` directory to build a docker image. + +Then use this latest code to build a docker image and deploy a submarine system in k8s. Then run test case in the `test-k8s/..` directory. + +### Run k8s test in locally + +Executing the following command will perform the following actions: + +``` +mvn -Phadoop-2.9 clean package install -DskipTests verify -DskipRat -am -pl submarine-test/test-k8s +``` + +1. The submarine project will be compiled and packaged to generate `submarine-dist/target/submarine-.tar.gz` +2. Call the `submarine-cloud/hack/integration-test.sh` script + + + Call the `build.sh` script under `submarine/dev-support/docker-images/` to generate the latest `submarine`, `database` and `operator` docker images. + + Call `submarine-cloud/hack/kind-cluster-build.sh` to create a k8s cluster + + Call `submarine-cloud/hack/deploy-submarine.sh` to deploy the submarine system in the k8s cluster using the latest `submarine`, `database` and `operator` docker images. + + Call the test cases in `submarine-test/test-k8s/` for testing. + +### Run k8s test in travis + +Each time a code is submitted, travis is automatically triggered for testing. + +## E2E test + +### E2E tests can be executed both locally and in Travis (For workbench developer) +* Run E2E tests locally: + * Step1: Follow [HowToRun.md](https://github.com/apache/submarine/blob/master/docs/workbench/HowToRun.md) to launch the submarine-server and database. + * Step2: Run workbench (Angular version) locally + ``` + cd submarine/submarine-workbench/workbench-web + npm start + // Check 127.0.0.1:4200 + ``` + * Step3: Modify the port from 8080 to 4200 + * [WebDriverManager.java](https://github.com/apache/submarine/blob/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/WebDriverManager.java): `url = "http://localhost:8080";` --> `url = "http://localhost:4200";` + * [Your Unit test case](https://github.com/apache/submarine/tree/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration): `8080` --> `4200` + * Step4: Comment the `headless` option + * [ChromeWebDriverProvider.java](https://github.com/apache/submarine/blob/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/ChromeWebDriverProvider.java): `chromeOptions.addArguments("--headless");` --> `//chromeOptions.addArguments("--headless");` + * With the `headless` option, the selenium will be executed in background. + * Step5: Run E2E test cases (Please check the following section **Run the existing tests**) +* Run E2E tests in Travis: + * Step1: Make sure that the port must be 8080 rather than in [WebDriverManager.java](https://github.com/apache/submarine/blob/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/WebDriverManager.java) and [all test cases](https://github.com/apache/submarine/tree/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration). + * Step2: Make sure that the `headless` option is not commented in [ChromeWebDriverProvider.java](https://github.com/apache/submarine/blob/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/ChromeWebDriverProvider.java). + * Step3: If you push the commit to Github, the Travis CI will execute automatically and you can check it in `https://travis-ci.org/${your_github_account}/${your_repo_name}`. +### Run the existing tests. +##### Move to the working directory. +``` +cd submarine/submarine-test/test-e2e +``` +##### Compile & Run. + +> Following command will compile all files and run **all** files ending with "IT" in the [directory](https://github.com/apache/submarine/tree/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration). +* For linux + ``` + mvn verify + ``` +* For MacOS +``` +mvn clean install -U +``` +> Run a specific testcase +``` +mvn -Dtest=${your_test_case_file_name} test //ex: mvn -Dtest=loginIT test +``` + +##### Result +If all of the function under test are succeeded, it will show. +``` +BUILD SUCCESS +``` +Otherwise, it will show. +``` +BUILD FAILURE +``` + +### Add your own integration test +1. Create a new file ending with "IT" under "submarine/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration/". +2. Your public class is recommended to extend AbstractSubmarineIT. The class AbstractSubmarineIT contains some commonly used functions. +```java + WebElement pollingWait(final By locator, final long timeWait); // Find element on the website. + void clickAndWait(final By locator); // Click element and wait for 1 second. + void sleep(long millis, boolean logOutput); // Let system sleep a period of time. +``` +3. There are also some commonly used functions except in AbstractSubmarineIT.java. +```java + // In WebDriverManager.java: + public static WebDriver getWebDriver(); // This return a firefox webdriver which has been set to your workbench website. +``` +4. Add [JUnit](https://junit.org/junit5/docs/current/user-guide/) annotation before your testing function, e.g., @Beforeclass, @Test, and @AfterClass. You can refer to [loginIT.java](https://github.com/apache/submarine/blob/master/submarine-test/test-e2e/src/test/java/org/apache/submarine/integration/loginIT.java). +5. Use command mentioned above to compile and run to test whether it works as your anticipation. + + diff --git a/docs/development/QuickDevTips.md b/docs/development/QuickDevTips.md new file mode 100644 index 0000000000000000000000000000000000000000..e614a46d9c225affc97d818d41de9238dbb53b6f --- /dev/null +++ b/docs/development/QuickDevTips.md @@ -0,0 +1,36 @@ +## 1. Introduction + +This document describes some useful tips that can accelerate your development efficiency. + +### Run server outside of k8s +If you are working on submarine-server, you do not need to bundle submarine-server into docker-image, and restart the helm-chart every time. + +You can run each module individually and only need to re-package the submarine-server, getting rid of other unnecessary process. + +1. Run db docker + +``` +docker run -it -p 3306:3306 -d --name submarine-database -e MYSQL_ROOT_PASSWORD=password apache/submarine:database-0.5.0-SNAPSHOT +``` + +2. Run k8s + +``` +minikube start # or other alternatives, such as kind +kubectl apply -f ./dev-support/k8s/tfjob/crd.yaml +kubectl kustomize ./dev-support/k8s/tfjob/operator | kubectl apply -f - +kubectl apply -f ./dev-support/k8s/pytorchjob/ +export KUBECONFIG=/home//.kube/config # (in ~/.bashrc) +``` +3. Package server + +``` +mvn clean package -DskipTests +``` + +4. Start server + +cd submarine-dist/target/submarine-dist-0.5.0-SNAPSHOT-hadoop-2.9/submarine-dist-0.5.0-SNAPSHOT-hadoop-2.9/ +./bin/submarine-daemon.sh start getMysqlJar +``` +``` \ No newline at end of file diff --git a/docs/development/README.md b/docs/development/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f917d7729e2b9b2aed4502bbccbf14dbe646bad9 --- /dev/null +++ b/docs/development/README.md @@ -0,0 +1,133 @@ + +# Development Guide + +## 1. Introduction + +This document mainly describes the structure of each module of the Submarine project, the development and test description of each module. + +## 2. Submarine Project Structure + +### 2.1. submarine-client +Provide the CLI interface for submarine user. (Currently only support YARN service) + +### 2.2. submarine-cloud +Define submarine operator. (Work in progress) + +### 2.3. submarine-commons +Define utility function used in multiple packages, mainly related to hadoop. + +### 2.4. submarine-dist +Store the pre-release files. + +### 2.5. submarine-sdk +Provide Python SDK for submarine user. + +### 2.6. submarine-security +Provide authorization for Apache Spark to talking to Ranger Admin. + +### 2.7. submarine-server +Include core server, restful api, and k8s/yarn submitter. + +### 2.8. submarine-test +Provide end-to-end and k8s test for submarine. +### 2.9. submarine-workbench + +- workbench-server: is a Jetty-based web server service. Workbench-server provides RESTful interface and Websocket interface. The RESTful interface provides workbench-web with management capabilities for databases such as project, department, user, and role. +- workbench-web: is a web front-end service based on Angular.js framework. With workbench-web users can manage Submarine project, department, user, role through browser. You can also use the notebook to develop machine learning algorithms, model release and other lifecycle management. +### 2.10 dev-support + ++ **mini-submarine**: by using the docker image provided by Submarine, you can +experience all the functions of Submarine in a single docker environment, while +mini-submarine also provides developers with a development and testing +environment, Avoid the hassle of installing and deploying the runtime +environment. ++ **submarine-installer**: submarine-installer is our submarine runtime +environment installation tool for yarn-3.1+ and above.By using +submarine-installer, it is easy to install and deploy system services such as +`docker`, `nvidia-docker`, `nvidia driver`, `ETCD`, `Calico network` etc. +required by yarn-3.1+. + +## 3. Submarine Workbench Development Guide + +Submarine workbench consists of three modules: workbench-server, workbench-web +and database. + +First, you need to clone the entire Submarine project: + +``` +git clone https://github.com/apache/submarine.git +``` + +## 3.1 Database + +Submarine selects mysql as the workbench database, and stores the table +structure and information of all the data in workbench in mysql. Please browse +the project's [Submarine Database Guide](../database/) documentation and follow +the instructions to install a mysql database via docker in your development and +test environment. + +## 3.2 Workbench-web + ++ **Install dependencies** + +You only need to execute the following command when you run workbench-web for +the first time, so you can follow the depend. environment of node.js in the +workbench-web directory. + +```bash +cd submarine-workbench/workbench-web +yarn install +``` + +The node.js dependency library will be installed in the node_modules directory. node_modules does not need to be uploaded to the git repository. We have excluded it in the submarine/.gitignore file. You can clean this by manually deleting the directory or executing mvn clean. table of Contents. + ++ **Compiles and hot-reloads for development** + +```bash +yarn run build +``` + +By executing the above command, `workbench-web` will publish the web page to the `workbench-web/dist` directory. Later we will also add the feature of `workbench-web` to a WAR package, which only requires a `workbench-web.war` file. Package can release submarine workbench. + ++ **Lints and fixes files** + +When you write the `Angular, js` file in `workbench-web` through IDEA, because IDEA can't format these files well, you need to execute the following command to format the angular and js files to avoid some warnings during the yarn build. + +``` +yarn run lint +``` + +In fact, you must execute this command when you develop and submit any workbench-web feature. Otherwise, chances are that you won't be able to check the code specification we set up in Travis. + +## 3.3 Workbench-server + +Workbench-server has a built-in jetty server service, so you don't need to install any web container service. You can start submarine workbench by launching `workbench-server` directly in IDEA. + ++ **Run / Debug** : In the IDEA, add a Run/Debug Configuration, `Main Class:` select `org.apache.submarine.server.SubmarineServer`, `Use classpath of module:` select `submarine-server-core`. + +![img](../assets/Run-Debug-Configuration.png) + +So you can pass http://127.0.0.1:8080 debugging or running to submarine-workbench. + ++ **It should be noted**: + +1. Because `workbench-web/dist` is the webapp default directory of the workbench-server Jetty Server, the workbench-server will automatically load the `workbench-web/dist` directory after startup. + +2. The `workbench-web/dist` used by `workbench-server` is used as the webapp directory, which is configured via `workbench-site.xml`, but we do not recommend you to modify it. + +3. The submarine-workbench, IP and 8080 ports that are accessible locally through the port 8080 of 127.0.0.1 are configured via `workbench-site.xml`, but we do not recommend you to modify it. + +4. When you modify the `angular` or `js` of workbench-web, you need to execute the yarn run build command in the workbench-web directory, and let your modified code update to the dist directory, so that you can see the effect of your code modification in the workbench. diff --git a/docs/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs.md b/docs/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs.md new file mode 100644 index 0000000000000000000000000000000000000000..2eb32b49cff68bb188071e088519bc03503296f8 --- /dev/null +++ b/docs/ecosystem/kaldi/RunningDistributedThchs30KaldiJobs.md @@ -0,0 +1,678 @@ + + +# Thchs30 Kaldi Example With YARN Service + +## Prepare data for training + +Thchs30 is a common benchmark in machine learning for speech data and transcripts. Below example is based on Thchs30 dataset. + +1) download gz file: +``` +THCHS30_PATH=/data/hdfs1/nfs/aisearch/kaldi/thchs30 +mkdir $THCHS30_PATH/data && cd $THCHS30_PATH/data +wget http://www.openslr.org/resources/18/data_thchs30.tgz +wget http://www.openslr.org/resources/18/test-noise.tgz +wget http://www.openslr.org/resources/18/resource.tgz +``` + +2) Checkout https://github.com/apache/submarine.git: +``` +git clone https://github.com/apache/submarine.git +``` + +3) Go to `submarine/docker/ecosystem/` +``` +cp -r ./kaldi/sge $THCHS30_PATH/sge +``` + +4) optional，Modify `/opt/kaldi/egs/thchs30/s5/cmd.sh` in the Container,This queue is used by default +``` +export train_cmd="queue.pl -q all.q" +``` + +**Warning:** + +Please note that YARN service doesn't allow multiple services with the same name, so please run following command +``` +yarn application -destroy +``` +to delete services if you want to reuse the same service name. + +## Prepare Docker images + +Refer to [Write Dockerfile](WriteDockerfileKaldi.md) to build a Docker image or use prebuilt one: + +- hadoopsubmarine/kaldi-latest-gpu-base:0.0.1 + +## Run Kaldi jobs + +### Run distributed training + +``` +# Change the variables according to your needs +SUBMARINE_VERSION=3.3.0-SNAPSHOT +WORKER_NUM=2 +SGE_CFG_PATH=/cfg +THCHS30_PATH=/data/hdfs1/nfs/aisearch/kaldi/thchs30 +DOCKER_HADOOP_HDFS_HOME=/app/${SUBMARINE_VERSION} + +# Dependent on registrydns, you must fill in < your RegistryDNSIP> in resolv.conf +yarn jar /usr/local/matrix/share/hadoop/yarn/${SUBMARINE_VERSION}.jar \ +job run --name kaldi-thchs30-distributed \ +--env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \ +--env DOCKER_HADOOP_HDFS_HOME=$DOCKER_HADOOP_HDFS_HOME \ +--env YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK=calico-network \ +--env PYTHONUNBUFFERED="0" \ +--env TZ="Asia/Shanghai" \ +--env YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS=${THCHS30_PATH}/sge/resolv.conf:/etc/resolv.conf,\ +${THCHS30_PATH}/sge/passwd:/etc/passwd:rw,\ +${THCHS30_PATH}/sge/group:/etc/group:rw,\ +${THCHS30_PATH}/sge:$SGE_CFG_PATH,\ +${THCHS30_PATH}/data:/opt/kaldi/egs/thchs30,\ +${THCHS30_PATH}/mul/s5:/opt/kaldi/egs/mul-thchs30/s5 \ +--input_path /opt/kaldi/egs/thchs30/data \ +--docker_image hadoopsubmarine/kaldi-latest-gpu-base:0.0.1 \ +--num_workers $WORKER_NUM \ +--worker_resources memory=64G,vcores=32,gpu=1 \ +--worker_launch_cmd "sudo mkdir -p /opt/kaldi/egs/mul-thchs30/s5 && \ +sudo cp /opt/kaldi/egs/thchs30/s5/* /opt/kaldi/egs/mul-thchs30/s5 -r && \ +cluster_user=`whoami` domain_suffix="ml.com" && \ +cd /cfg && bash sge_run.sh $WORKER_NUM $SGE_CFG_PATH && \ +if [ $(echo $HOST_NAME |grep "^master-") ] then sleep 2m && cd /opt/kaldi/egs/mul-thchs30/s5 && ./run.sh fi" \ +--verbose +``` + +Explanations: + +- `>1` num_workers indicates it is a distributed training. +- Parameters / resources / Docker image of parameter server can be specified separately. For many cases, parameter server doesn't require GPU.We don't need parameter server here + +For the meaning of the individual parameters, see the [QuickStart](project/github/submarine/docs/userdocs/yarn/QuickStart.md) page! + +*Outputs of distributed training* + +Sample output of master: +``` +... +Reading package lists... +Building dependency tree... +Reading state information... +The following additional packages will be installed: + bsd-mailx cpio gridengine-common ifupdown iproute2 isc-dhcp-client + isc-dhcp-common libatm1 libdns-export162 libisc-export160 liblockfile-bin + liblockfile1 libmnl0 libxmuu1 libxtables11 ncurses-term netbase + openssh-client openssh-server openssh-sftp-server postfix python3-chardet + python3-pkg-resources python3-requests python3-six python3-urllib3 + ssh-import-id ssl-cert tcsh xauth +Suggested packages: + libarchive1 gridengine-qmon ppp rdnssd iproute2-doc resolvconf avahi-autoipd + isc-dhcp-client-ddns apparmor ssh-askpass libpam-ssh keychain monkeysphere + rssh molly-guard ufw procmail postfix-mysql postfix-pgsql postfix-ldap + postfix-pcre sasl2-bin libsasl2-modules dovecot-common postfix-cdb + postfix-doc python3-setuptools python3-ndg-httpsclient python3-openssl + python3-pyasn1 openssl-blacklist +The following NEW packages will be installed: + bsd-mailx cpio gridengine-client gridengine-common gridengine-exec + gridengine-master ifupdown iproute2 isc-dhcp-client isc-dhcp-common libatm1 + libdns-export162 libisc-export160 liblockfile-bin liblockfile1 libmnl0 + libxmuu1 libxtables11 ncurses-term netbase openssh-client openssh-server + openssh-sftp-server postfix python3-chardet python3-pkg-resources + python3-requests python3-six python3-urllib3 ssh-import-id ssl-cert tcsh + xauth +0 upgraded, 33 newly installed, 0 to remove and 30 not upgraded. +Need to get 12.1 MB of archives. +After this operation, 65.8 MB of additional disk space will be used. +Get:1 http://archive.ubuntu.com/ubuntu xenial/main amd64 libatm1 amd64 1:2.5.1-1.5 [24.2 kB] +Get:2 http://archive.ubuntu.com/ubuntu xenial/main amd64 libmnl0 amd64 1.0.3-5 [12.0 kB] +Get:3 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile-bin amd64 1.09-6ubuntu1 [10.8 kB] +Get:4 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile1 amd64 1.09-6ubuntu1 [8056 B] +Get:5 http://archive.ubuntu.com/ubuntu xenial/main amd64 cpio amd64 2.11+dfsg-5ubuntu1 [74.8 kB] +Get:6 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 iproute2 amd64 4.3.0-1ubuntu3.16.04.5 [523 kB] +Get:7 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 ifupdown amd64 0.8.10ubuntu1.4 [54.9 kB] +Get:8 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libisc-export160 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [153 kB] +Get:9 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libdns-export162 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [665 kB] +Get:10 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-client amd64 4.3.3-5ubuntu12.10 [224 kB] +Get:11 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-common amd64 4.3.3-5ubuntu12.10 [105 kB] +Get:12 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxtables11 amd64 1.6.0-2ubuntu3 [27.2 kB] +Get:13 http://archive.ubuntu.com/ubuntu xenial/main amd64 netbase all 5.3 [12.9 kB] +Get:14 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxmuu1 amd64 2:1.1.2-2 [9674 B] +Get:15 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-client amd64 1:7.2p2-4ubuntu2.8 [590 kB] +Get:16 http://archive.ubuntu.com/ubuntu xenial/main amd64 xauth amd64 1:1.0.9-1ubuntu2 [22.7 kB] +Get:17 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssl-cert all 1.0.37 [16.9 kB] +Get:18 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 postfix amd64 3.1.0-3ubuntu0.3 [1152 kB] +Get:19 http://archive.ubuntu.com/ubuntu xenial/main amd64 bsd-mailx amd64 8.1.2-0.20160123cvs-2 [63.7 kB] +Get:20 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-common all 6.2u5-7.4 [156 kB] +Get:21 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-client amd64 6.2u5-7.4 [3394 kB] +Get:22 http://archive.ubuntu.com/ubuntu xenial/universe amd64 tcsh amd64 6.18.01-5 [410 kB] +Get:23 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-exec amd64 6.2u5-7.4 [990 kB] +Get:24 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-master amd64 6.2u5-7.4 [2429 kB] +Get:25 http://archive.ubuntu.com/ubuntu xenial/main amd64 ncurses-term all 6.0+20160213-1ubuntu1 [249 kB] +Get:26 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-sftp-server amd64 1:7.2p2-4ubuntu2.8 [38.9 kB] +Get:27 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-server amd64 1:7.2p2-4ubuntu2.8 [335 kB] +Get:28 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-pkg-resources all 20.7.0-1 [79.0 kB] +Get:29 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-chardet all 2.3.0-2 [96.2 kB] +Get:30 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-six all 1.10.0-3 [11.0 kB] +Get:31 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-urllib3 all 1.13.1-2ubuntu0.16.04.3 [58.5 kB] +Get:32 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-requests all 2.9.1-3ubuntu0.1 [55.8 kB] +Get:33 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssh-import-id all 5.5-0ubuntu1 [10.2 kB] +Fetched 12.1 MB in 0s (15.0 MB/s) +Selecting previously unselected package libatm1:amd64. +(Reading database ... +(Reading database ... 5% +(Reading database ... 10% +(Reading database ... 15% +(Reading database ... 20% +(Reading database ... 25% +(Reading database ... 30% +(Reading database ... 35% +(Reading database ... 40% +(Reading database ... 45% +(Reading database ... 50% +(Reading database ... 55% +(Reading database ... 60% +(Reading database ... 65% +(Reading database ... 70% +(Reading database ... 75% +(Reading database ... 80% +(Reading database ... 85% +(Reading database ... 90% +(Reading database ... 95% +(Reading database ... 100% +(Reading database ... 21398 files and directories currently installed.) +Preparing to unpack .../libatm1_1%3a2.5.1-1.5_amd64.deb ... +Unpacking libatm1:amd64 (1:2.5.1-1.5) ... +Selecting previously unselected package libmnl0:amd64. +Preparing to unpack .../libmnl0_1.0.3-5_amd64.deb ... +Unpacking libmnl0:amd64 (1.0.3-5) ... +Selecting previously unselected package liblockfile-bin. +Preparing to unpack .../liblockfile-bin_1.09-6ubuntu1_amd64.deb ... +Unpacking liblockfile-bin (1.09-6ubuntu1) ... +Selecting previously unselected package liblockfile1:amd64. +Preparing to unpack .../liblockfile1_1.09-6ubuntu1_amd64.deb ... +Unpacking liblockfile1:amd64 (1.09-6ubuntu1) ... +Selecting previously unselected package cpio. +Preparing to unpack .../cpio_2.11+dfsg-5ubuntu1_amd64.deb ... +Unpacking cpio (2.11+dfsg-5ubuntu1) ... +Selecting previously unselected package iproute2. +Preparing to unpack .../iproute2_4.3.0-1ubuntu3.16.04.5_amd64.deb ... +Unpacking iproute2 (4.3.0-1ubuntu3.16.04.5) ... +Selecting previously unselected package ifupdown. +Preparing to unpack .../ifupdown_0.8.10ubuntu1.4_amd64.deb ... +Unpacking ifupdown (0.8.10ubuntu1.4) ... +Selecting previously unselected package libisc-export160. +Preparing to unpack .../libisc-export160_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ... +Unpacking libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Selecting previously unselected package libdns-export162. +Preparing to unpack .../libdns-export162_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ... +Unpacking libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Selecting previously unselected package isc-dhcp-client. +Preparing to unpack .../isc-dhcp-client_4.3.3-5ubuntu12.10_amd64.deb ... +Unpacking isc-dhcp-client (4.3.3-5ubuntu12.10) ... +Selecting previously unselected package isc-dhcp-common. +Preparing to unpack .../isc-dhcp-common_4.3.3-5ubuntu12.10_amd64.deb ... +Unpacking isc-dhcp-common (4.3.3-5ubuntu12.10) ... +Selecting previously unselected package libxtables11:amd64. +Preparing to unpack .../libxtables11_1.6.0-2ubuntu3_amd64.deb ... +Unpacking libxtables11:amd64 (1.6.0-2ubuntu3) ... +Selecting previously unselected package netbase. +Preparing to unpack .../archives/netbase_5.3_all.deb ... +Unpacking netbase (5.3) ... +Selecting previously unselected package libxmuu1:amd64. +Preparing to unpack .../libxmuu1_2%3a1.1.2-2_amd64.deb ... +Unpacking libxmuu1:amd64 (2:1.1.2-2) ... +Selecting previously unselected package openssh-client. +Preparing to unpack .../openssh-client_1%3a7.2p2-4ubuntu2.8_amd64.deb ... +Unpacking openssh-client (1:7.2p2-4ubuntu2.8) ... +Selecting previously unselected package xauth. +Preparing to unpack .../xauth_1%3a1.0.9-1ubuntu2_amd64.deb ... +Unpacking xauth (1:1.0.9-1ubuntu2) ... +Selecting previously unselected package ssl-cert. +Preparing to unpack .../ssl-cert_1.0.37_all.deb ... +Unpacking ssl-cert (1.0.37) ... +Selecting previously unselected package postfix. +Preparing to unpack .../postfix_3.1.0-3ubuntu0.3_amd64.deb ... +Unpacking postfix (3.1.0-3ubuntu0.3) ... +Selecting previously unselected package bsd-mailx. +Preparing to unpack .../bsd-mailx_8.1.2-0.20160123cvs-2_amd64.deb ... +Unpacking bsd-mailx (8.1.2-0.20160123cvs-2) ... +Selecting previously unselected package gridengine-common. +Preparing to unpack .../gridengine-common_6.2u5-7.4_all.deb ... +Unpacking gridengine-common (6.2u5-7.4) ... +Selecting previously unselected package gridengine-client. +Preparing to unpack .../gridengine-client_6.2u5-7.4_amd64.deb ... +Unpacking gridengine-client (6.2u5-7.4) ... +Selecting previously unselected package tcsh. +Preparing to unpack .../tcsh_6.18.01-5_amd64.deb ... +Unpacking tcsh (6.18.01-5) ... +Selecting previously unselected package gridengine-exec. +Preparing to unpack .../gridengine-exec_6.2u5-7.4_amd64.deb ... +Unpacking gridengine-exec (6.2u5-7.4) ... +Selecting previously unselected package gridengine-master. +Preparing to unpack .../gridengine-master_6.2u5-7.4_amd64.deb ... +Unpacking gridengine-master (6.2u5-7.4) ... +Selecting previously unselected package ncurses-term. +Preparing to unpack .../ncurses-term_6.0+20160213-1ubuntu1_all.deb ... +Unpacking ncurses-term (6.0+20160213-1ubuntu1) ... +Selecting previously unselected package openssh-sftp-server. +Preparing to unpack .../openssh-sftp-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ... +Unpacking openssh-sftp-server (1:7.2p2-4ubuntu2.8) ... +Selecting previously unselected package openssh-server. +Preparing to unpack .../openssh-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ... +Unpacking openssh-server (1:7.2p2-4ubuntu2.8) ... +Selecting previously unselected package python3-pkg-resources. +Preparing to unpack .../python3-pkg-resources_20.7.0-1_all.deb ... +Unpacking python3-pkg-resources (20.7.0-1) ... +Selecting previously unselected package python3-chardet. +Preparing to unpack .../python3-chardet_2.3.0-2_all.deb ... +Unpacking python3-chardet (2.3.0-2) ... +Selecting previously unselected package python3-six. +Preparing to unpack .../python3-six_1.10.0-3_all.deb ... +Unpacking python3-six (1.10.0-3) ... +Selecting previously unselected package python3-urllib3. +Preparing to unpack .../python3-urllib3_1.13.1-2ubuntu0.16.04.3_all.deb ... +Unpacking python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ... +Selecting previously unselected package python3-requests. +Preparing to unpack .../python3-requests_2.9.1-3ubuntu0.1_all.deb ... +Unpacking python3-requests (2.9.1-3ubuntu0.1) ... +Selecting previously unselected package ssh-import-id. +Preparing to unpack .../ssh-import-id_5.5-0ubuntu1_all.deb ... +Unpacking ssh-import-id (5.5-0ubuntu1) ... +Processing triggers for systemd (229-4ubuntu21.22) ... +Processing triggers for libc-bin (2.23-0ubuntu11) ... +Setting up libatm1:amd64 (1:2.5.1-1.5) ... +Setting up libmnl0:amd64 (1.0.3-5) ... +Setting up liblockfile-bin (1.09-6ubuntu1) ... +Setting up liblockfile1:amd64 (1.09-6ubuntu1) ... +Setting up cpio (2.11+dfsg-5ubuntu1) ... +update-alternatives: using /bin/mt-gnu to provide /bin/mt (mt) in auto mode +Setting up iproute2 (4.3.0-1ubuntu3.16.04.5) ... +Setting up ifupdown (0.8.10ubuntu1.4) ... +Creating /etc/network/interfaces. +Setting up libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Setting up libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Setting up isc-dhcp-client (4.3.3-5ubuntu12.10) ... +Setting up isc-dhcp-common (4.3.3-5ubuntu12.10) ... +Setting up libxtables11:amd64 (1.6.0-2ubuntu3) ... +Setting up netbase (5.3) ... +Setting up libxmuu1:amd64 (2:1.1.2-2) ... +Setting up openssh-client (1:7.2p2-4ubuntu2.8) ... +Setting up xauth (1:1.0.9-1ubuntu2) ... +Setting up ssl-cert (1.0.37) ... +Setting up postfix (3.1.0-3ubuntu0.3) ... +Creating /etc/postfix/dynamicmaps.cf +setting myhostname: master-0.XXX +setting alias maps +setting alias database +changing /etc/mailname to master-0.XXX +setting myorigin +setting destinations: $myhostname, master-0.XXX, localhost.XXX, , localhost +setting relayhost: +setting mynetworks: 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +setting mailbox_size_limit: 0 +setting recipient_delimiter: + +setting inet_interfaces: all +setting inet_protocols: all +/etc/aliases does not exist, creating it. +WARNING: /etc/aliases exists, but does not have a root alias. + +Postfix is now set up with a default configuration. If you need to make +changes, edit +/etc/postfix/main.cf (and others) as needed. To view Postfix configuration +values, see postconf(1). + +After modifying main.cf, be sure to run '/etc/init.d/postfix reload'. + +Running newaliases +invoke-rc.d: could not determine current runlevel +invoke-rc.d: policy-rc.d denied execution of restart. +Setting up bsd-mailx (8.1.2-0.20160123cvs-2) ... +update-alternatives: using /usr/bin/bsd-mailx to provide /usr/bin/mailx (mailx) in auto mode +Setting up gridengine-common (6.2u5-7.4) ... + +Creating config file /etc/default/gridengine with new version +Setting up gridengine-client (6.2u5-7.4) ... +Setting up tcsh (6.18.01-5) ... +update-alternatives: using /bin/tcsh to provide /bin/csh (csh) in auto mode +Setting up gridengine-exec (6.2u5-7.4) ... +invoke-rc.d: could not determine current runlevel +invoke-rc.d: policy-rc.d denied execution of start. +Setting up gridengine-master (6.2u5-7.4) ... +su: Authentication failure +(Ignored) +Initializing cluster with the following parameters: + => SGE_ROOT: /var/lib/gridengine + => SGE_CELL: default + => Spool directory: /var/spool/gridengine/spooldb + => Initial manager user: sgeadmin +Initializing spool (/var/spool/gridengine/spooldb) +Initializing global configuration based on /usr/share/gridengine/default-configuration +Initializing complexes based on /usr/share/gridengine/centry +Initializing usersets based on /usr/share/gridengine/usersets +Adding user sgeadmin as a manager +Cluster creation complete +invoke-rc.d: could not determine current runlevel +invoke-rc.d: policy-rc.d denied execution of start. +Setting up ncurses-term (6.0+20160213-1ubuntu1) ... +Setting up openssh-sftp-server (1:7.2p2-4ubuntu2.8) ... +Setting up openssh-server (1:7.2p2-4ubuntu2.8) ... +Creating SSH2 RSA key; this may take some time ... +2048 SHA256:hfQpES1aS4cjF8AOCIParZR6342vdwutoyITru0wtuE root@master-0.XXX (RSA) +Creating SSH2 DSA key; this may take some time ... +1024 SHA256:gOsPMVgwXBHJzixN/gtJAG+hVCHqw8t7Fhy4nsx8od0 root@master-0.XXX (DSA) +Creating SSH2 ECDSA key; this may take some time ... +256 SHA256:3D5SNniUb4z+/BuqXheFgG+DfjsxXqTT/zwWAqdX4jM root@master-0.XXX (ECDSA) +Creating SSH2 ED25519 key; this may take some time ... +256 SHA256:SwyeV9iSqOW4TKLi4Wvc0zD8lWtupHCJpDu8oWBwbfU root@master-0.XXX (ED25519) +invoke-rc.d: could not determine current runlevel +invoke-rc.d: policy-rc.d denied execution of start. +Setting up python3-pkg-resources (20.7.0-1) ... +Setting up python3-chardet (2.3.0-2) ... +Setting up python3-six (1.10.0-3) ... +Setting up python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ... +Setting up python3-requests (2.9.1-3ubuntu0.1) ... +Setting up ssh-import-id (5.5-0ubuntu1) ... +Processing triggers for libc-bin (2.23-0ubuntu11) ... +Processing triggers for systemd (229-4ubuntu21.22) ... +Reading package lists... +Building dependency tree... +Reading state information... +0 upgraded, 0 newly installed, 0 to remove and 30 not upgraded. +``` + +cat $SGE_CFG_PATH/setcfg.log +``` +finish master +add worker node worker-0.XXX +``` + +Sample output of worker: +``` +please wait +Reading package lists... +Building dependency tree... +Reading state information... +The following additional packages will be installed: + bsd-mailx cpio gridengine-common ifupdown iproute2 isc-dhcp-client + isc-dhcp-common libatm1 libdns-export162 libisc-export160 liblockfile-bin + liblockfile1 libmnl0 libxmuu1 libxtables11 ncurses-term netbase + openssh-client openssh-server openssh-sftp-server postfix python3-chardet + python3-pkg-resources python3-requests python3-six python3-urllib3 + ssh-import-id ssl-cert tcsh xauth +Suggested packages: + libarchive1 gridengine-qmon ppp rdnssd iproute2-doc resolvconf avahi-autoipd + isc-dhcp-client-ddns apparmor ssh-askpass libpam-ssh keychain monkeysphere + rssh molly-guard ufw procmail postfix-mysql postfix-pgsql postfix-ldap + postfix-pcre sasl2-bin libsasl2-modules dovecot-common postfix-cdb + postfix-doc python3-setuptools python3-ndg-httpsclient python3-openssl + python3-pyasn1 openssl-blacklist +The following NEW packages will be installed: + bsd-mailx cpio gridengine-client gridengine-common gridengine-exec ifupdown + iproute2 isc-dhcp-client isc-dhcp-common libatm1 libdns-export162 + libisc-export160 liblockfile-bin liblockfile1 libmnl0 libxmuu1 libxtables11 + ncurses-term netbase openssh-client openssh-server openssh-sftp-server + postfix python3-chardet python3-pkg-resources python3-requests python3-six + python3-urllib3 ssh-import-id ssl-cert tcsh xauth +0 upgraded, 32 newly installed, 0 to remove and 30 not upgraded. +Need to get 9633 kB of archives. +After this operation, 51.2 MB of additional disk space will be used. +Get:1 http://archive.ubuntu.com/ubuntu xenial/main amd64 libatm1 amd64 1:2.5.1-1.5 [24.2 kB] +Get:2 http://archive.ubuntu.com/ubuntu xenial/main amd64 libmnl0 amd64 1.0.3-5 [12.0 kB] +Get:3 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile-bin amd64 1.09-6ubuntu1 [10.8 kB] +Get:4 http://archive.ubuntu.com/ubuntu xenial/main amd64 liblockfile1 amd64 1.09-6ubuntu1 [8056 B] +Get:5 http://archive.ubuntu.com/ubuntu xenial/main amd64 cpio amd64 2.11+dfsg-5ubuntu1 [74.8 kB] +Get:6 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 iproute2 amd64 4.3.0-1ubuntu3.16.04.5 [523 kB] +Get:7 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 ifupdown amd64 0.8.10ubuntu1.4 [54.9 kB] +Get:8 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libisc-export160 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [153 kB] +Get:9 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 libdns-export162 amd64 1:9.10.3.dfsg.P4-8ubuntu1.15 [665 kB] +Get:10 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-client amd64 4.3.3-5ubuntu12.10 [224 kB] +Get:11 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 isc-dhcp-common amd64 4.3.3-5ubuntu12.10 [105 kB] +Get:12 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxtables11 amd64 1.6.0-2ubuntu3 [27.2 kB] +Get:13 http://archive.ubuntu.com/ubuntu xenial/main amd64 netbase all 5.3 [12.9 kB] +Get:14 http://archive.ubuntu.com/ubuntu xenial/main amd64 libxmuu1 amd64 2:1.1.2-2 [9674 B] +Get:15 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-client amd64 1:7.2p2-4ubuntu2.8 [590 kB] +Get:16 http://archive.ubuntu.com/ubuntu xenial/main amd64 xauth amd64 1:1.0.9-1ubuntu2 [22.7 kB] +Get:17 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssl-cert all 1.0.37 [16.9 kB] +Get:18 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 postfix amd64 3.1.0-3ubuntu0.3 [1152 kB] +Get:19 http://archive.ubuntu.com/ubuntu xenial/main amd64 bsd-mailx amd64 8.1.2-0.20160123cvs-2 [63.7 kB] +Get:20 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-common all 6.2u5-7.4 [156 kB] +Get:21 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-client amd64 6.2u5-7.4 [3394 kB] +Get:22 http://archive.ubuntu.com/ubuntu xenial/universe amd64 tcsh amd64 6.18.01-5 [410 kB] +Get:23 http://archive.ubuntu.com/ubuntu xenial/universe amd64 gridengine-exec amd64 6.2u5-7.4 [990 kB] +Get:24 http://archive.ubuntu.com/ubuntu xenial/main amd64 ncurses-term all 6.0+20160213-1ubuntu1 [249 kB] +Get:25 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-sftp-server amd64 1:7.2p2-4ubuntu2.8 [38.9 kB] +Get:26 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 openssh-server amd64 1:7.2p2-4ubuntu2.8 [335 kB] +Get:27 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-pkg-resources all 20.7.0-1 [79.0 kB] +Get:28 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-chardet all 2.3.0-2 [96.2 kB] +Get:29 http://archive.ubuntu.com/ubuntu xenial/main amd64 python3-six all 1.10.0-3 [11.0 kB] +Get:30 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-urllib3 all 1.13.1-2ubuntu0.16.04.3 [58.5 kB] +Get:31 http://archive.ubuntu.com/ubuntu xenial-updates/main amd64 python3-requests all 2.9.1-3ubuntu0.1 [55.8 kB] +Get:32 http://archive.ubuntu.com/ubuntu xenial/main amd64 ssh-import-id all 5.5-0ubuntu1 [10.2 kB] +Fetched 9633 kB in 2s (4496 kB/s) +Selecting previously unselected package libatm1:amd64. +(Reading database ... +(Reading database ... 5% +(Reading database ... 10% +(Reading database ... 15% +(Reading database ... 20% +(Reading database ... 25% +(Reading database ... 30% +(Reading database ... 35% +(Reading database ... 40% +(Reading database ... 45% +(Reading database ... 50% +(Reading database ... 55% +(Reading database ... 60% +(Reading database ... 65% +(Reading database ... 70% +(Reading database ... 75% +(Reading database ... 80% +(Reading database ... 85% +(Reading database ... 90% +(Reading database ... 95% +(Reading database ... 100% +(Reading database ... 21398 files and directories currently installed.) +Preparing to unpack .../libatm1_1%3a2.5.1-1.5_amd64.deb ... +Unpacking libatm1:amd64 (1:2.5.1-1.5) ... +Selecting previously unselected package libmnl0:amd64. +Preparing to unpack .../libmnl0_1.0.3-5_amd64.deb ... +Unpacking libmnl0:amd64 (1.0.3-5) ... +Selecting previously unselected package liblockfile-bin. +Preparing to unpack .../liblockfile-bin_1.09-6ubuntu1_amd64.deb ... +Unpacking liblockfile-bin (1.09-6ubuntu1) ... +Selecting previously unselected package liblockfile1:amd64. +Preparing to unpack .../liblockfile1_1.09-6ubuntu1_amd64.deb ... +Unpacking liblockfile1:amd64 (1.09-6ubuntu1) ... +Selecting previously unselected package cpio. +Preparing to unpack .../cpio_2.11+dfsg-5ubuntu1_amd64.deb ... +Unpacking cpio (2.11+dfsg-5ubuntu1) ... +Selecting previously unselected package iproute2. +Preparing to unpack .../iproute2_4.3.0-1ubuntu3.16.04.5_amd64.deb ... +Unpacking iproute2 (4.3.0-1ubuntu3.16.04.5) ... +Selecting previously unselected package ifupdown. +Preparing to unpack .../ifupdown_0.8.10ubuntu1.4_amd64.deb ... +Unpacking ifupdown (0.8.10ubuntu1.4) ... +Selecting previously unselected package libisc-export160. +Preparing to unpack .../libisc-export160_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ... +Unpacking libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Selecting previously unselected package libdns-export162. +Preparing to unpack .../libdns-export162_1%3a9.10.3.dfsg.P4-8ubuntu1.15_amd64.deb ... +Unpacking libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Selecting previously unselected package isc-dhcp-client. +Preparing to unpack .../isc-dhcp-client_4.3.3-5ubuntu12.10_amd64.deb ... +Unpacking isc-dhcp-client (4.3.3-5ubuntu12.10) ... +Selecting previously unselected package isc-dhcp-common. +Preparing to unpack .../isc-dhcp-common_4.3.3-5ubuntu12.10_amd64.deb ... +Unpacking isc-dhcp-common (4.3.3-5ubuntu12.10) ... +Selecting previously unselected package libxtables11:amd64. +Preparing to unpack .../libxtables11_1.6.0-2ubuntu3_amd64.deb ... +Unpacking libxtables11:amd64 (1.6.0-2ubuntu3) ... +Selecting previously unselected package netbase. +Preparing to unpack .../archives/netbase_5.3_all.deb ... +Unpacking netbase (5.3) ... +Selecting previously unselected package libxmuu1:amd64. +Preparing to unpack .../libxmuu1_2%3a1.1.2-2_amd64.deb ... +Unpacking libxmuu1:amd64 (2:1.1.2-2) ... +Selecting previously unselected package openssh-client. +Preparing to unpack .../openssh-client_1%3a7.2p2-4ubuntu2.8_amd64.deb ... +Unpacking openssh-client (1:7.2p2-4ubuntu2.8) ... +Selecting previously unselected package xauth. +Preparing to unpack .../xauth_1%3a1.0.9-1ubuntu2_amd64.deb ... +Unpacking xauth (1:1.0.9-1ubuntu2) ... +Selecting previously unselected package ssl-cert. +Preparing to unpack .../ssl-cert_1.0.37_all.deb ... +Unpacking ssl-cert (1.0.37) ... +Selecting previously unselected package postfix. +Preparing to unpack .../postfix_3.1.0-3ubuntu0.3_amd64.deb ... +Unpacking postfix (3.1.0-3ubuntu0.3) ... +Selecting previously unselected package bsd-mailx. +Preparing to unpack .../bsd-mailx_8.1.2-0.20160123cvs-2_amd64.deb ... +Unpacking bsd-mailx (8.1.2-0.20160123cvs-2) ... +Selecting previously unselected package gridengine-common. +Preparing to unpack .../gridengine-common_6.2u5-7.4_all.deb ... +Unpacking gridengine-common (6.2u5-7.4) ... +Selecting previously unselected package gridengine-client. +Preparing to unpack .../gridengine-client_6.2u5-7.4_amd64.deb ... +Unpacking gridengine-client (6.2u5-7.4) ... +Selecting previously unselected package tcsh. +Preparing to unpack .../tcsh_6.18.01-5_amd64.deb ... +Unpacking tcsh (6.18.01-5) ... +Selecting previously unselected package gridengine-exec. +Preparing to unpack .../gridengine-exec_6.2u5-7.4_amd64.deb ... +Unpacking gridengine-exec (6.2u5-7.4) ... +Selecting previously unselected package ncurses-term. +Preparing to unpack .../ncurses-term_6.0+20160213-1ubuntu1_all.deb ... +Unpacking ncurses-term (6.0+20160213-1ubuntu1) ... +Selecting previously unselected package openssh-sftp-server. +Preparing to unpack .../openssh-sftp-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ... +Unpacking openssh-sftp-server (1:7.2p2-4ubuntu2.8) ... +Selecting previously unselected package openssh-server. +Preparing to unpack .../openssh-server_1%3a7.2p2-4ubuntu2.8_amd64.deb ... +Unpacking openssh-server (1:7.2p2-4ubuntu2.8) ... +Selecting previously unselected package python3-pkg-resources. +Preparing to unpack .../python3-pkg-resources_20.7.0-1_all.deb ... +Unpacking python3-pkg-resources (20.7.0-1) ... +Selecting previously unselected package python3-chardet. +Preparing to unpack .../python3-chardet_2.3.0-2_all.deb ... +Unpacking python3-chardet (2.3.0-2) ... +Selecting previously unselected package python3-six. +Preparing to unpack .../python3-six_1.10.0-3_all.deb ... +Unpacking python3-six (1.10.0-3) ... +Selecting previously unselected package python3-urllib3. +Preparing to unpack .../python3-urllib3_1.13.1-2ubuntu0.16.04.3_all.deb ... +Unpacking python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ... +Selecting previously unselected package python3-requests. +Preparing to unpack .../python3-requests_2.9.1-3ubuntu0.1_all.deb ... +Unpacking python3-requests (2.9.1-3ubuntu0.1) ... +Selecting previously unselected package ssh-import-id. +Preparing to unpack .../ssh-import-id_5.5-0ubuntu1_all.deb ... +Unpacking ssh-import-id (5.5-0ubuntu1) ... +Processing triggers for systemd (229-4ubuntu21.22) ... +Processing triggers for libc-bin (2.23-0ubuntu11) ... +Setting up libatm1:amd64 (1:2.5.1-1.5) ... +Setting up libmnl0:amd64 (1.0.3-5) ... +Setting up liblockfile-bin (1.09-6ubuntu1) ... +Setting up liblockfile1:amd64 (1.09-6ubuntu1) ... +Setting up cpio (2.11+dfsg-5ubuntu1) ... +update-alternatives: using /bin/mt-gnu to provide /bin/mt (mt) in auto mode +Setting up iproute2 (4.3.0-1ubuntu3.16.04.5) ... +Setting up ifupdown (0.8.10ubuntu1.4) ... +Creating /etc/network/interfaces. +Setting up libisc-export160 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Setting up libdns-export162 (1:9.10.3.dfsg.P4-8ubuntu1.15) ... +Setting up isc-dhcp-client (4.3.3-5ubuntu12.10) ... +Setting up isc-dhcp-common (4.3.3-5ubuntu12.10) ... +Setting up libxtables11:amd64 (1.6.0-2ubuntu3) ... +Setting up netbase (5.3) ... +Setting up libxmuu1:amd64 (2:1.1.2-2) ... +Setting up openssh-client (1:7.2p2-4ubuntu2.8) ... +Setting up xauth (1:1.0.9-1ubuntu2) ... +Setting up ssl-cert (1.0.37) ... +Setting up postfix (3.1.0-3ubuntu0.3) ... +Creating /etc/postfix/dynamicmaps.cf +setting myhostname: worker-0.XXX +setting alias maps +setting alias database +changing /etc/mailname to worker-0.XXX +setting myorigin +setting destinations: $myhostname, worker-0.XXX, localhost.XXX, , localhost +setting relayhost: +setting mynetworks: 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +setting mailbox_size_limit: 0 +setting recipient_delimiter: + +setting inet_interfaces: all +setting inet_protocols: all +/etc/aliases does not exist, creating it. +WARNING: /etc/aliases exists, but does not have a root alias. + +Postfix is now set up with a default configuration. If you need to make +changes, edit +/etc/postfix/main.cf (and others) as needed. To view Postfix configuration +values, see postconf(1). + +After modifying main.cf, be sure to run '/etc/init.d/postfix reload'. + +Running newaliases +invoke-rc.d: could not determine current runlevel +invoke-rc.d: policy-rc.d denied execution of restart. +Setting up bsd-mailx (8.1.2-0.20160123cvs-2) ... +update-alternatives: using /usr/bin/bsd-mailx to provide /usr/bin/mailx (mailx) in auto mode +Setting up gridengine-common (6.2u5-7.4) ... + +Creating config file /etc/default/gridengine with new version +Setting up gridengine-client (6.2u5-7.4) ... +Setting up tcsh (6.18.01-5) ... +update-alternatives: using /bin/tcsh to provide /bin/csh (csh) in auto mode +Setting up gridengine-exec (6.2u5-7.4) ... +invoke-rc.d: could not determine current runlevel +invoke-rc.d: policy-rc.d denied execution of start. +Setting up ncurses-term (6.0+20160213-1ubuntu1) ... +Setting up openssh-sftp-server (1:7.2p2-4ubuntu2.8) ... +Setting up openssh-server (1:7.2p2-4ubuntu2.8) ... +Creating SSH2 RSA key; this may take some time ... +2048 SHA256:ok/TxzwtF5W8I55sDxrt4Agy4fuWn39BiSovvDObhVE root@worker-0.XXX (RSA) +Creating SSH2 DSA key; this may take some time ... +1024 SHA256:4y48kVYt3mS3q1KgZzEoYMnS/2d/tA8TJUK5uNSaxZY root@worker-0.XXX (DSA) +Creating SSH2 ECDSA key; this may take some time ... +256 SHA256:4D7zm4cD2IbDnHoXnzcIo3FISbvOW8eOstGBNf1/bvo root@worker-0.XXX (ECDSA) +Creating SSH2 ED25519 key; this may take some time ... +256 SHA256:/HrA3xiZiH5CZkXwtcfE6GwcMM+hEhZzTdFHxj4PzDg root@worker-0.XXX (ED25519) +invoke-rc.d: could not determine current runlevel +invoke-rc.d: policy-rc.d denied execution of start. +Setting up python3-pkg-resources (20.7.0-1) ... +Setting up python3-chardet (2.3.0-2) ... +Setting up python3-six (1.10.0-3) ... +Setting up python3-urllib3 (1.13.1-2ubuntu0.16.04.3) ... +Setting up python3-requests (2.9.1-3ubuntu0.1) ... +Setting up ssh-import-id (5.5-0ubuntu1) ... +Processing triggers for libc-bin (2.23-0ubuntu11) ... +Processing triggers for systemd (229-4ubuntu21.22) ... +Reading package lists... +Building dependency tree... +Reading state information... +0 upgraded, 0 newly installed, 0 to remove and 30 not upgraded. +``` + +cat $SGE_CFG_PATH/setcfg.log +``` +please wait +Start SGE for worker is finished +done for worker-0.XXX worker. +``` + +Sample output of sge: +![alt text](pictures/sge_cluster.png "SGE for multiple jobs") + +![alt text](pictures/sge_stat.png "SGE for user multiple jobs") diff --git a/docs/ecosystem/kaldi/WriteDockerfileKaldi.md b/docs/ecosystem/kaldi/WriteDockerfileKaldi.md new file mode 100644 index 0000000000000000000000000000000000000000..8e0449bd1223a4ebdff87ccd5c47d7a709ea3e04 --- /dev/null +++ b/docs/ecosystem/kaldi/WriteDockerfileKaldi.md @@ -0,0 +1,112 @@ + + +## Creating Docker Images for Running Kaldi on YARN + +### How to create docker images to run Kaldi on YARN + +Dockerfile to run Kaldi on YARN need two part: + +**Base libraries which Kaldi depends on** + +1) OS base image, for example ```nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04``` + +2) Kaldi depended libraries and packages. For example ```python```, ```g++```, ```make```. For GPU support, need ```cuda```, ```cudnn```, etc. + +3) Kaldi compile. + +**Libraries to access HDFS** + +1) JDK + +2) Hadoop + +Here's an example of a base image (w/o GPU support) to install Kaldi: +```shell +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 + +RUN apt-get clean && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + sudo \ + openjdk-8-jdk \ + iputils-ping \ + g++ \ + make \ + automake \ + autoconf \ + bzip2 \ + unzip \ + wget \ + sox \ + libtool \ + git \ + subversion \ + python2.7 \ + python3 \ + zlib1g-dev \ + ca-certificates \ + patch \ + ffmpeg \ + vim && \ + rm -rf /var/lib/apt/lists/* && \ + ln -s /usr/bin/python2.7 /usr/bin/python + +RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \ + cd /opt/kaldi && \ + cd /opt/kaldi/tools && \ + ./extras/install_mkl.sh && \ + make -j $(nproc) && \ + cd /opt/kaldi/src && \ + ./configure --shared --use-cuda && \ + make depend -j $(nproc) && \ + make -j $(nproc) +``` + +On top of above image, add files, install packages to access HDFS +```shell +RUN apt-get update && apt-get install -y openjdk-8-jdk wget +# Install hadoop +ENV HADOOP_VERSION="3.2.1" +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \ + tar zxf hadoop-${HADOOP_VERSION}.tar.gz && \ + ln -s hadoop-${HADOOP_VERSION} hadoop-current && \ + rm hadoop-${HADOOP_VERSION}.tar.gz +``` + +Build and push to your own docker registry: Use ```docker build ... ``` and ```docker push ...``` to finish this step. + +### Use examples to build your own Kaldi docker images + +We provided following examples for you to build kaldi docker images. + +For latest Kaldi + +- *base/ubuntu-18.04/Dockerfile.gpu.kaldi_latest: Latest Kaldi that supports GPU, which is prebuilt to CUDA10, with models. + +### Build Docker images + +#### Manually build Docker image: + +Under `docker/` directory,The CLUSTER_NAME can be modified in build-all.sh to have installation permissions, run `build-all.sh` to build Docker images. It will build following images: + +- `kaldi-latest-gpu-base:0.0.1` for base Docker image which includes Hadoop, Kaldi, GPU base libraries, which includes thchs30 model. + +#### Use prebuilt images + +(No liability) +You can also use prebuilt images for convenience in the docker hub: +- hadoopsubmarine/kaldi-latest-gpu-base:0.0.1 diff --git a/docs/ecosystem/kaldi/base/ubuntu-18.04/Dockerfile.gpu.kaldi_latest b/docs/ecosystem/kaldi/base/ubuntu-18.04/Dockerfile.gpu.kaldi_latest new file mode 100644 index 0000000000000000000000000000000000000000..44e2ae298945a9c00741fcd158d317f5f1d017ba --- /dev/null +++ b/docs/ecosystem/kaldi/base/ubuntu-18.04/Dockerfile.gpu.kaldi_latest @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 + +RUN apt-get clean && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + sudo \ + openjdk-8-jdk \ + iputils-ping \ + g++ \ + make \ + automake \ + autoconf \ + bzip2 \ + unzip \ + wget \ + sox \ + libtool \ + git \ + subversion \ + python2.7 \ + python3 \ + zlib1g-dev \ + ca-certificates \ + patch \ + ffmpeg \ + vim && \ + rm -rf /var/lib/apt/lists/* && \ + ln -s /usr/bin/python2.7 /usr/bin/python + +RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi && \ + cd /opt/kaldi && \ + cd /opt/kaldi/tools && \ + ./extras/install_mkl.sh && \ + make -j $(nproc) && \ + cd /opt/kaldi/src && \ + ./configure --shared --use-cuda && \ + make depend -j $(nproc) && \ + make -j $(nproc) + +WORKDIR / +# Install Hadoop +ENV HADOOP_VERSION="3.2.1" +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +ARG CLUSTER_NAME=submarine +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \ + tar zxf hadoop-${HADOOP_VERSION}.tar.gz && \ + ln -s hadoop-${HADOOP_VERSION} hadoop-current && \ + rm hadoop-${HADOOP_VERSION}.tar.gz + +RUN echo "Install python related packages" && \ + pip --no-cache-dir install Pillow h5py ipykernel jupyter matplotlib numpy pandas scipy sklearn && \ + python -m ipykernel.kernelspec + +ENV CLUSTER_NAME="admin" # Your cluster user sets root privileges +RUN echo "## Allow root to run any commands anywhere" >> /etc/sudoers && \ + echo "User_Alias USER_ROOT = tf-docker, $CLUSTER_NAME" >> /etc/sudoers && \ + echo "root ALL=(ALL) ALL" >> /etc/sudoers && \ + echo "USER_ROOT ALL=(ALL) ALL" >> /etc/sudoers && \ + echo "USER_ROOT ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ No newline at end of file diff --git a/docs/ecosystem/kaldi/build-all.sh b/docs/ecosystem/kaldi/build-all.sh new file mode 100755 index 0000000000000000000000000000000000000000..61c590bbf89a5f41ffb14f3838a41eb8c0270e4a --- /dev/null +++ b/docs/ecosystem/kaldi/build-all.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Building base images" + +set -euo pipefail + +cd base/ubuntu-18.04 + +docker build --build-arg CLUSTER_NAME=test . -f Dockerfile.gpu.kaldi_latest -t kaldi-latest-gpu-base:0.0.1 + +echo "Finished building base images" diff --git a/docs/ecosystem/kaldi/pictures/sge_cluster.png b/docs/ecosystem/kaldi/pictures/sge_cluster.png new file mode 100644 index 0000000000000000000000000000000000000000..9db22ee5d98f5349b735a3f525fed5666f1206d2 Binary files /dev/null and b/docs/ecosystem/kaldi/pictures/sge_cluster.png differ diff --git a/docs/ecosystem/kaldi/pictures/sge_stat.png b/docs/ecosystem/kaldi/pictures/sge_stat.png new file mode 100644 index 0000000000000000000000000000000000000000..1e1c92b37f48bfa0571b2729cf6d629e7bd3e0cf Binary files /dev/null and b/docs/ecosystem/kaldi/pictures/sge_stat.png differ diff --git a/docs/ecosystem/kaldi/sge/gencfs.sh b/docs/ecosystem/kaldi/sge/gencfs.sh new file mode 100644 index 0000000000000000000000000000000000000000..4e5482d75acadd52e9337b3b6d9d954c39b80904 --- /dev/null +++ b/docs/ecosystem/kaldi/sge/gencfs.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +WORKER_HOST_STRS=$1 +SLOTS=$2 + +cat << EOF > /tmp/qconf-mc.txt +arch a RESTRING == YES NO NONE 0 +calendar c RESTRING == YES NO NONE 0 +cpu cpu DOUBLE >= YES NO 0 0 +display_win_gui dwg BOOL == YES NO 0 0 +gpu g INT <= YES YES 0 10000 +h_core h_core MEMORY <= YES NO 0 0 +h_cpu h_cpu TIME <= YES NO 0:0:0 0 +h_data h_data MEMORY <= YES NO 0 0 +h_fsize h_fsize MEMORY <= YES NO 0 0 +h_rss h_rss MEMORY <= YES NO 0 0 +h_rt h_rt TIME <= YES NO 0:0:0 0 +h_stack h_stack MEMORY <= YES NO 0 0 +h_vmem h_vmem MEMORY <= YES NO 0 0 +hostname h HOST == YES NO NONE 0 +load_avg la DOUBLE >= NO NO 0 0 +load_long ll DOUBLE >= NO NO 0 0 +load_medium lm DOUBLE >= NO NO 0 0 +load_short ls DOUBLE >= NO NO 0 0 +m_core core INT <= YES NO 0 0 +m_socket socket INT <= YES NO 0 0 +m_topology topo RESTRING == YES NO NONE 0 +m_topology_inuse utopo RESTRING == YES NO NONE 0 +mem_free mf MEMORY <= YES NO 0 0 +mem_total mt MEMORY <= YES NO 0 0 +mem_used mu MEMORY >= YES NO 0 0 +min_cpu_interval mci TIME <= NO NO 0:0:0 0 +np_load_avg nla DOUBLE >= NO NO 0 0 +np_load_long nll DOUBLE >= NO NO 0 0 +np_load_medium nlm DOUBLE >= NO NO 0 0 +np_load_short nls DOUBLE >= NO NO 0 0 +num_proc p INT == YES NO 0 0 +qname q RESTRING == YES NO NONE 0 +ram_free ram_free MEMORY <= YES JOB 0 0 +rerun re BOOL == NO NO 0 0 +s_core s_core MEMORY <= YES NO 0 0 +s_cpu s_cpu TIME <= YES NO 0:0:0 0 +s_data s_data MEMORY <= YES NO 0 0 +s_fsize s_fsize MEMORY <= YES NO 0 0 +s_rss s_rss MEMORY <= YES NO 0 0 +s_rt s_rt TIME <= YES NO 0:0:0 0 +s_stack s_stack MEMORY <= YES NO 0 0 +s_vmem s_vmem MEMORY <= YES NO 0 0 +seq_no seq INT == NO NO 0 0 +slots s INT <= YES YES 1 1000 +swap_free sf MEMORY <= YES NO 0 0 +swap_rate sr MEMORY >= YES NO 0 0 +swap_rsvd srsv MEMORY >= YES NO 0 0 +swap_total st MEMORY <= YES NO 0 0 +swap_used su MEMORY >= YES NO 0 0 +tmpdir tmp RESTRING == NO NO NONE 0 +virtual_free vf MEMORY <= YES NO 0 0 +virtual_total vt MEMORY <= YES NO 0 0 +virtual_used vu MEMORY >= YES NO 0 0 +EOF + +cat << EOF > /tmp/qconf-ae.txt +hostname $(hostname) +load_scaling NONE +complex_values ram_free=160G,gpu=2 +user_lists NONE +xuser_lists NONE +projects NONE +xprojects NONE +usage_scaling NONE +report_variables NONE +EOF + +cat << EOS > /tmp/qconf-ap.txt +pe_name smp +slots 32 +user_lists NONE +xuser_lists NONE +start_proc_args /bin/true +stop_proc_args /bin/true +allocation_rule \$pe_slots +control_slaves FALSE +job_is_first_task TRUE +urgency_slots min +accounting_summary FALSE +EOS + +cat << EOF > /tmp/qconf-aq.txt +qname all.q +hostlist $WORKER_HOST_STRS +seq_no 0 +load_thresholds np_load_avg=1.75 +suspend_thresholds NONE +nsuspend 1 +suspend_interval 00:05:00 +priority 0 +min_cpu_interval 00:05:00 +processors UNDEFINED +qtype BATCH INTERACTIVE +ckpt_list NONE +pe_list make smp +rerun FALSE +slots $SLOTS +tmpdir /tmp +shell /bin/bash +prolog NONE +epilog NONE +shell_start_mode posix_compliant +starter_method NONE +suspend_method NONE +resume_method NONE +terminate_method NONE +notify 00:00:60 +owner_list NONE +user_lists NONE +xuser_lists NONE +subordinate_list NONE +complex_values NONE +projects NONE +xprojects NONE +calendar NONE +initial_state default +s_rt INFINITY +h_rt INFINITY +s_cpu INFINITY +h_cpu INFINITY +s_fsize INFINITY +h_fsize INFINITY +s_data INFINITY +h_data INFINITY +s_stack INFINITY +h_stack INFINITY +s_core INFINITY +h_core INFINITY +s_rss INFINITY +h_rss INFINITY +s_vmem INFINITY +h_vmem INFINITY +EOF diff --git a/docs/ecosystem/kaldi/sge/group b/docs/ecosystem/kaldi/sge/group new file mode 100644 index 0000000000000000000000000000000000000000..88c6ba8e37afd95b0f7b8765530886958c6f54c1 --- /dev/null +++ b/docs/ecosystem/kaldi/sge/group @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +root:x:0: +daemon:x:1: +bin:x:2: +sys:x:3: +adm:x:4: +tty:x:5: +disk:x:6: +lp:x:7: +mail:x:8: +news:x:9: +uucp:x:10: +man:x:12: +proxy:x:13: +kmem:x:15: +dialout:x:20: +fax:x:21: +voice:x:22: +cdrom:x:24: +floppy:x:25: +tape:x:26: +sudo:x:27: +audio:x:29: +dip:x:30: +www-data:x:33: +backup:x:34: +operator:x:37: +list:x:38: +irc:x:39: +src:x:40: +gnats:x:41: +shadow:x:42: +utmp:x:43: +video:x:44: +sasl:x:45: +plugdev:x:46: +staff:x:50: +games:x:60: +users:x:100: +nogroup:x:65534: +systemd-journal:x:101: +systemd-timesync:x:102: +systemd-network:x:103: +systemd-resolve:x:104: +systemd-bus-proxy:x:106: +etcd:x:1099: +netdev:x:105: +ssh:x:107: +ssl-cert:x:108: +postfix:x:109: +postdrop:x:110: +sgeadmin:x:111: diff --git a/docs/ecosystem/kaldi/sge/passwd b/docs/ecosystem/kaldi/sge/passwd new file mode 100644 index 0000000000000000000000000000000000000000..a39047da29d70625c30f71d4cf52e1b3b6a0a796 --- /dev/null +++ b/docs/ecosystem/kaldi/sge/passwd @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +sys:x:3:3:sys:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +games:x:5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin +mail:x:8:8:mail:/var/mail:/usr/sbin/nologin +news:x:9:9:news:/var/spool/news:/usr/sbin/nologin +uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin +proxy:x:13:13:proxy:/bin:/usr/sbin/nologin +www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin +backup:x:34:34:backup:/var/backups:/usr/sbin/nologin +list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin +irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin +gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin +systemd-timesync:x:100:102:systemd Time Synchronization,,,:/run/systemd:/bin/false +systemd-network:x:101:103:systemd Network Management,,,:/run/systemd/netif:/bin/false +systemd-resolve:x:102:104:systemd Resolver,,,:/run/systemd/resolve:/bin/false +systemd-bus-proxy:x:103:105:systemd Bus Proxy,,,:/run/systemd:/bin/false +_apt:x:104:65534::/nonexistent:/bin/false +etcd:x:1099:1099::/home/etcd: +postfix:x:105:109::/var/spool/postfix:/bin/false +sgeadmin:x:106:111::/var/lib/gridengine:/bin/false +sshd:x:107:65534::/var/run/sshd:/usr/sbin/nologin diff --git a/docs/ecosystem/kaldi/sge/resolv.conf b/docs/ecosystem/kaldi/sge/resolv.conf new file mode 100644 index 0000000000000000000000000000000000000000..e51b279e369870eb992c1bbac9d65a17c59e9f64 --- /dev/null +++ b/docs/ecosystem/kaldi/sge/resolv.conf @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default system properties included when running spark-submit. +# This is useful for setting default environmental settings. + +nameserver 127.0.0.11 +nameserver < your RegistryDNSIP> +options timeout:1 ndots:0 diff --git a/docs/ecosystem/kaldi/sge/sge_run.sh b/docs/ecosystem/kaldi/sge/sge_run.sh new file mode 100644 index 0000000000000000000000000000000000000000..d2076ee904453b62035a6bb7fa0705968b1689a8 --- /dev/null +++ b/docs/ecosystem/kaldi/sge/sge_run.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Execute command format : sge_run.sh %num_workers% %input_path% +if [ $# -ne 2 ] +then + echo "Parameter setting error!" + exit 1 +fi + +WORKER_NUMBER=$1 +CFG_PATH=$2 + +HOST_NAME=$(hostname) +JOB_NAME=`echo $HOST_NAME |awk -F "." '{print($2)}'` +CLUSTER_USER=`echo $HOST_NAME |awk -F "." '{print($3)}'` +DOMAIN_SUFFIX=`echo $HOST_NAME |awk -F "." '{for(i=4;i<=NF;i++){if(tostr == ""){tostr=$i}else{tostr=tostr"."$i};if(i==NF)print tostr}}'` + +MASTER_HOST="" +WORKER_HOST_STRS="" +declare -a WORKER_HOST_LIST +SLOTS="30" + +for ((num=0; num<$WORKER_NUMBER; num++)) +do + if [ $num -eq 0 ] + then + MASTER_HOST="master-0."$JOB_NAME"."$CLUSTER_USER"."$DOMAIN_SUFFIX + WORKER_HOST_STRS="master-0."$JOB_NAME"."$CLUSTER_USER"."$DOMAIN_SUFFIX + SLOTS=$SLOTS",[master-0."$JOB_NAME"."$CLUSTER_USER"."$DOMAIN_SUFFIX"=48]" + else + let tmpnum=$num-1 + WORKER_HOST_STRS=$WORKER_HOST_STRS",worker-"$tmpnum"."$JOB_NAME"."$CLUSTER_USER"."$DOMAIN_SUFFIX + WORKER_HOST_LIST+=("worker-"$tmpnum"."$JOB_NAME"."$CLUSTER_USER"."$DOMAIN_SUFFIX) + SLOTS=$SLOTS",[worker-"$tmpnum"."$JOB_NAME"."$CLUSTER_USER"."$DOMAIN_SUFFIX"=48]" + fi +done + +if [ $(echo $HOST_NAME |grep "^master-") ] +then + sudo su - -c "sleep 30s && export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install -y gridengine-master gridengine-exec gridengine-client;apt-get autoremove -y && apt-get clean && \ + /etc/init.d/gridengine-master start && /etc/init.d/gridengine-exec start" + + sudo su - -s /bin/bash -c ". ${CFG_PATH}/gencfs.sh $WORKER_HOST_STRS $SLOTS" + sudo su -c " qconf -Mc /tmp/qconf-mc.txt && qconf -Ae /tmp/qconf-ae.txt && qconf -as \`hostname\` && + qconf -Ap /tmp/qconf-ap.txt && qconf -Aq /tmp/qconf-aq.txt && qconf -am $CLUSTER_USER && + echo finish master >> ${CFG_PATH}/setcfg.log " + for worker_num in ${WORKER_HOST_LIST[@]} + do + echo add $worker_num + sudo su -c " qconf -ah $worker_num && echo add worker node $worker_num >> ${CFG_PATH}/setcfg.log " + done + +elif [ $(echo $HOST_NAME |grep "^worker-") ] +then + sudo su - -s /bin/bash -c "sleep 2m && echo please wait && echo please wait >> ${CFG_PATH}/setcfg.log" + sudo su - -c "export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install -y gridengine-client gridengine-exec; apt-get autoremove -y && apt-get clean" + sudo su - -c "echo $MASTER_HOST > /var/lib/gridengine/default/common/act_qmaster" + sudo su - -c "/etc/init.d/gridengine-exec start && echo Start SGE for worker is finished >> ${CFG_PATH}/setcfg.log" + + sudo su - -s /bin/bash -c ". ${CFG_PATH}/gencfs.sh $WORKER_HOST_STRS $SLOTS" + sudo su - -c "qconf -Me /tmp/qconf-ae.txt && echo done for $HOST_NAME worker. >> ${CFG_PATH}/setcfg.log" + +else + echo "hostname doesn't match! should start with master or worker!" 1>&2 exit 1 +fi + +#sudo su - -s /bin/bash -c "echo sge start to finish >> ${CFG_PATH}/setcfg.log" diff --git a/docs/submarine-sdk/README.md b/docs/submarine-sdk/README.md new file mode 100644 index 0000000000000000000000000000000000000000..207aadcee416d000642e8e85c006919aabfd06fa --- /dev/null +++ b/docs/submarine-sdk/README.md @@ -0,0 +1,28 @@ + + +# Submarine-SDK + +## Summary +- Support Python, Scala, R language for algorithm development + +- Support tracking/metrics APIs which allows developers +add tracking/metrics and view tracking/metrics from Submarine Workbench UI. + +### Python +- [pysubmarine](pysubmarine) + + + + diff --git a/docs/submarine-sdk/pysubmarine/development.md b/docs/submarine-sdk/pysubmarine/development.md new file mode 100644 index 0000000000000000000000000000000000000000..c7eaf94c44c1863864a99f77305e235607ce0f92 --- /dev/null +++ b/docs/submarine-sdk/pysubmarine/development.md @@ -0,0 +1,100 @@ + + +## Python Development +This page provides general Python development guidelines and source build instructions +### Prerequisites +This is required for developing & testing changes, we recommend installing pysubmarine +in its own conda environment by running the following +```bash +conda create --name submarine-dev python=3.6 +conda activate submarine-dev + +# lint-requirements.txt and test-requirements.txt are in ./submarine-sdk/pysubmarine/github-actions +pip install -r lint-requirements.txt +pip install -r test-requirements.txt + +# Installs pysubmarine from current checkout +pip install ./submarine-sdk/pysubmarine +``` +### PySubmarine Docker +We also use docker to provide build environments for CI, development, +generate python sdk from swagger. +```bash +./run-pysubmarine-ci.sh +``` +The script does the following things: +- Start an interactive bash session +- Mount submarine directory to /workspace and set it as home +- Switch user to be the same user that calls the `run-pysubmarine-ci.sh` +### Coding Style +- Use [yapf](https://github.com/google/yapf) to format Python code +- yapf style is configured in `.style.yapf` file +- To autoformat code +```bash +./submarine-sdk/pysubmarine/github-actions/auto-format.sh +``` +- Verify linter pass before submitting a pull request by running: +```bash +./submarine-sdk/pysubmarine/github-actions/lint.sh +``` +### Unit Testing +We are using [pytest](https://docs.pytest.org/en/latest/) to develop our unit test suite. +After building the project (see below) you can run its unit tests like so: +```bash +cd submarine-sdk/pysubmarine +``` +- Run unit test +```shell script +pytest --cov=submarine -vs -m "not e2e" +``` +- Run integration test +```shell script +pytest --cov=submarine -vs -m "e2e" +``` +> Before run this command in local, you should make sure the submarine server is running. + +### Generate python SDK from swagger +We use [open-api generator](https://openapi-generator.tech/docs/installation/#jar) +to generate pysubmarine client API that used to communicate with submarine server. + +If change below files, please run `./dev-support/pysubmarine/gen-sdk.sh` +to generate latest version of SDK. +- [Bootstrap.java](https://github.com/apache/submarine/blob/master/submarine-server/server-core/src/main/java/org/apache/submarine/server/Bootstrap.java) +- [ExperimentRestApi.java](https://github.com/apache/submarine/blob/master/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java) + +### Upload package to PyPi +For Apache Submarine committer and PMCs to do a new release. +1. Change the version from 0.x.x-SNAPSHOT to 0.x.x +in [setup.py](https://github.com/apache/submarine/blob/master/submarine-sdk/pysubmarine/setup.py) +2. Install Python packages +```bash +cd submarine-sdk/pysubmarine +pip install -r github-actions/pypi-requirements.txt +``` +3. Compiling Your Package + +It will create `build`, `dist`, and `project.egg.info` +in your local directory +```bash +python setup.py bdist_wheel +``` +4. Upload python package to TestPyPI for testing +```bash +python -m twine upload --repository testpypi dist/* +``` +5. Upload python package to PyPi +```bash +python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/* +``` diff --git a/docs/submarine-sdk/pysubmarine/tracking.md b/docs/submarine-sdk/pysubmarine/tracking.md new file mode 100644 index 0000000000000000000000000000000000000000..0c019b0c0eae801a163295201143e2137bcfba41 --- /dev/null +++ b/docs/submarine-sdk/pysubmarine/tracking.md @@ -0,0 +1,71 @@ + + +# pysubmarine-tracking +It helps developers use submarine's internal data caching, +data exchange, and task tracking capabilities to more efficiently improve the +development and execution of machine learning productivity +- Allow data scientist to track distributed ML job +- Support store ML parameters and metrics in Submarine-server +- Support hdfs,S3 and mysql (Currently we only support mysql) + +## Quickstart +1. [Start mini-submarine](../../../dev-support/mini-submarine#run-mini-submarine-image) + +2. [Start Mysql server in mini-submarine](../../../dev-support/mini-submarine#run-workbench-server) + +3. Uncomment the log_param and log_metric in +[mnist_distributed.py](../../../dev-support/mini-submarine/submarine/mnist_distributed.py) + +4. Start Submarine job (e.g., run_submarine_mnist_tony.sh) + +## Functions +### `submarine.get_tracking_uri()` + +return the tracking URI. + +### `submarine.set_tracking_uri(URI)` + +set the tracking URI. You can also set the +SUBMARINE_TRACKING_URI environment variable to have Submarine find a URI from +there. The URI should be database connection string. + +**Parameters** + +- URI - Submarine record data to Mysql server. The database URL +is expected in the format ``+://:@:/``. +By default it's `mysql+pymysql://submarine:password@localhost:3306/submarine`. +More detail : [SQLAlchemy docs](https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls) + + + +### `submarine.log_metric(key, value, step=0)` + +logs a single key-value metric. The value must always be a number. + +**Parameters** +- key - Metric name (string). +- value - Metric value (float). +- step - A single integer step at which to log the specified Metrics, +by default it's 0. + +### `submarine.log_param(key, value)` + +logs a single key-value parameter. The key and value are both strings. + +**Parameters** +- key - Parameter name (string). +- value - Parameter value (string). diff --git a/docs/submarine-security/spark-security/README.md b/docs/submarine-security/spark-security/README.md new file mode 100644 index 0000000000000000000000000000000000000000..33d905d8b955826486dbd0ee7578927dc57a132a --- /dev/null +++ b/docs/submarine-security/spark-security/README.md @@ -0,0 +1,134 @@ + + +# Submarine Spark Security Plugin + +ACL Management for Apache Spark SQL with Apache Ranger, enabling: + +- Table/Column level authorization +- Row level filtering +- Data masking + + +Security is one of fundamental features for enterprise adoption. [Apache Ranger™](https://ranger.apache.org) offers many security plugins for many Hadoop ecosystem components, +such as HDFS, Hive, HBase, Solr and Sqoop2. However, [Apache Spark™](http://spark.apache.org) is not counted in yet. +When a secured HDFS cluster is used as a data warehouse accessed by various users and groups via different applications wrote by Spark and Hive, +it is very difficult to guarantee data management in a consistent way. Apache Spark users visit data warehouse only +with Storage based access controls offered by HDFS. This library enables Spark with SQL Standard Based Authorization. + +## Build + +Please refer to the online documentation - [Building submarine spark security plguin](build-submarine-spark-security-plugin.md) + +## Quick Start + +Three steps to integrate Apache Spark and Apache Ranger. + +### Installation + +Place the submarine-spark-security-<version>.jar into `$SPARK_HOME/jars`. + +### Configurations + +#### Settings for Apache Ranger + +Create `ranger-spark-security.xml` in `$SPARK_HOME/conf` and add the following configurations +for pointing to the right Apache Ranger admin server. + + +```xml + + + + + ranger.plugin.spark.policy.rest.url + ranger admin address like http://ranger-admin.org:6080 + + + + ranger.plugin.spark.service.name + a ranger hive service name + + + + ranger.plugin.spark.policy.cache.dir + ./a ranger hive service name/policycache + + + + ranger.plugin.spark.policy.pollIntervalMs + 5000 + + + + ranger.plugin.spark.policy.source.impl + org.apache.ranger.admin.client.RangerAdminRESTClient + + + +``` + +Create `ranger-spark-audit.xml` in `$SPARK_HOME/conf` and add the following configurations +to enable/disable auditing. + +```xml + + + + xasecure.audit.is.enabled + true + + + + xasecure.audit.destination.db + false + + + + xasecure.audit.destination.db.jdbc.driver + com.mysql.jdbc.Driver + + + + xasecure.audit.destination.db.jdbc.url + jdbc:mysql://10.171.161.78/ranger + + + + xasecure.audit.destination.db.password + rangeradmin + + + + xasecure.audit.destination.db.user + rangeradmin + + + + +``` + +#### Settings for Apache Spark + +You can configure `spark.sql.extensions` with the `*Extension` we provided. +For example, `spark.sql.extensions=org.apache.submarine.spark.security.api.RangerSparkAuthzExtension` + +Currently, you can set the following options to `spark.sql.extensions` to choose authorization w/ or w/o +extra functions. + +| option | authorization | row filtering | data masking | +|---|---|---|---| +|org.apache.submarine.spark.security.api.RangerSparkAuthzExtension| √ | × | × | +|org.apache.submarine.spark.security.api.RangerSparkSQLExtension| √ | √ | √ | diff --git a/docs/submarine-security/spark-security/build-submarine-spark-security-plugin.md b/docs/submarine-security/spark-security/build-submarine-spark-security-plugin.md new file mode 100644 index 0000000000000000000000000000000000000000..ff3011230dcde90a9c1e3c8f4d815664ecb19bda --- /dev/null +++ b/docs/submarine-security/spark-security/build-submarine-spark-security-plugin.md @@ -0,0 +1,30 @@ + + +# Building Submarine Spark Security Plugin + +Submarine Spark Security Plugin is built using [Apache Maven](http://maven.apache.org). To build it, `cd` to the root direct of submarine project and run: + +```bash +mvn clean package -Dmaven.javadoc.skip=true -DskipTests -pl :submarine-spark-security +``` + +By default, Submarine Spark Security Plugin is built against Apache Spark 2.3.x and Apache Ranger 1.1.0, which may be incompatible with other Apache Spark or Apache Ranger releases. + +Currently, available profiles are: + +Spark: -Pspark-2.3, -Pspark-2.4, -Pspark-3.0 + +Ranger: -Pranger-1.0, -Pranger-1.1, -Pranger-1.2 -Pranger-2.0 diff --git a/docs/user-guide-home.md b/docs/user-guide-home.md new file mode 100644 index 0000000000000000000000000000000000000000..fc575d80eb82e3edce3a8291a0bb445213788277 --- /dev/null +++ b/docs/user-guide-home.md @@ -0,0 +1,29 @@ + +# User Document of Submarine + +This is index of user-document of Submarine. + +## Build From Code + +Follow the guidee [Build From Code Guide](./development/BuildFromCode.md) to build source code / binary distribution of Submarine, and rebuild Docker images. + +## Use Submarine on YARN + +Please refer to [Submarine On YARN](userdocs/yarn/README.md) + +## Use Submarine on K8s + +Please refer to [Submarine On K8s](userdocs/k8s/README.md) diff --git a/docs/userdocs/k8s/README.md b/docs/userdocs/k8s/README.md new file mode 100644 index 0000000000000000000000000000000000000000..63f5326c7dca5a398cd8e3189af8ab121569551e --- /dev/null +++ b/docs/userdocs/k8s/README.md @@ -0,0 +1,67 @@ + + +# Submarine on K8s +Submarine for K8s supports distributed TensorFlow and PyTorch. + +Submarine can run on K8s >= 1.14, supports features like GPU isolation. + +We have validated Submarine on the following versions: + +| K8s Version | Support? | +| ------------- |:-------------:| +| 1.13.x (or earlier) | X | +| 1.14.x | √ | +| 1.15.x | √ | +| 1.16.x | √ | +| 1.17.x | To be verified | +| 1.17.x | To be verified | + +## Install Submarine + +### Setup Kubernetes +Submarine can be deployed on any K8s environment if version matches. If you don't have a running K8s, you can set up a K8s using [Docker Desktop](https://www.docker.com/products/docker-desktop), [MiniKube](https://kubernetes.io/docs/tasks/tools/install-minikube/), or [kind, Kubernetes-in-Docker](https://kind.sigs.k8s.io/). + +From our experiences, Docker Desktop is an easier choice. + +### Install Submarine Use Helm Charts +After you have an up-and-running K8s, you can follow [Submarine Helm Charts Guide](helm.md) to deploy Submarine services on K8s cluster in minutes. + +## Using Jupyter notebooks in Submarine + +Please refer to [Notebook guide](notebook.md) if you want to use jupyter notebook in Submarine. + +## Model Training (Experiment) on K8s + +### Model Training with Submarine UI + +You can now submit Submarine model training on the UI, please refer to [Submit Experiment from Submarine UI](./run-tensorflow-experiment-ui.md) + +### Model Training With Submarine Python SDK + +You can also follow the guidance to run Submarine model training using Python SDK, please refer to [Submit Experiment using Python SDK](./run-experiment-python-sdk.md) + +### Model Training With Submarine REST API + +Alternatively, we support use REST API to submit, list, delete experiments (model training) + +- [Run model training using Tensorflow](run-tensorflow-experiment-rest.md) +- [Run model training using PyTorch](run-pytorch-experiment-rest.md) +- [Experiment API Reference](api/experiment.md) + diff --git a/docs/userdocs/k8s/api/environment.md b/docs/userdocs/k8s/api/environment.md new file mode 100644 index 0000000000000000000000000000000000000000..1b4138d16e8eefe84b89d4319d5999be7af9b111 --- /dev/null +++ b/docs/userdocs/k8s/api/environment.md @@ -0,0 +1,227 @@ + + +# Environment API Reference + +> Note: The Environment API is in the alpha stage which is subjected to incompatible changes in +> future releases. + +## Create Environment +`POST /api/v1/environment` + +**Example Request** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "name": "my-submarine-env", + "dockerImage" : "continuumio/anaconda3", + "kernelSpec" : { + "name" : "team_default_python_3.7", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0", + "anaconda=2020.02=py37_0", + "anaconda-client=1.7.2=py37_0", + "anaconda-navigator=1.9.12=py37_0"] + } +} +' http://127.0.0.1:8080/api/v1/environment +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "environmentId": "environment_1586156073228_0001", + "environmentSpec": { + "name": "my-submarine-env", + "dockerImage" : "continuumio/anaconda3", + "kernelSpec" : { + "name" : "team_default_python_3.7", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0", + "anaconda=2020.02=py37_0", + "anaconda-client=1.7.2=py37_0", + "anaconda-navigator=1.9.12=py37_0"] + } + } + } +} +``` + +### List environment +`GET /api/v1/environment` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/environment +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": [ + { + "environmentId": "environment_1586156073228_0001", + "environmentSpec": { + "name": "my-submarine-env", + "dockerImage" : "continuumio/anaconda3", + "kernelSpec" : { + "name" : "team_default_python_3.7", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0", + "anaconda=2020.02=py37_0", + "anaconda-client=1.7.2=py37_0", + "anaconda-navigator=1.9.12=py37_0"] + } + } + }, + { + "environmentId": "environment_1586156073228_0002", + "environmentSpec": { + "name": "my-submarine-env-2", + "dockerImage" : "continuumio/miniconda", + "kernelSpec" : { + "name" : "team_miniconda_python_3.7", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0"] + } + } + } + ] +} +``` + +### Get environment +`GET /api/v1/environment/{name}` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/environment/my-submarine-env +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "environmentId": "environment_1586156073228_0001", + "environmentSpec": { + "name": "my-submarine-env", + "dockerImage" : "continuumio/anaconda3", + "kernelSpec" : { + "name" : "team_default_python_3.7", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0", + "anaconda=2020.02=py37_0", + "anaconda-client=1.7.2=py37_0", + "anaconda-navigator=1.9.12=py37_0"] + } + } + } +} +``` + +### Patch environment +`PATCH /api/v1/environment/{name}` + +**Example Request:** +```sh +curl -X PATCH -H "Content-Type: application/json" -d ' +{ + "name": "my-submarine-env", + "dockerImage" : "continuumio/anaconda3", + "kernelSpec" : { + "name" : "team_default_python_3.7_updated", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0"] + } +} +' http://127.0.0.1:8080/api/v1/environment/my-submarine-env +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "success": true, + "result": { + "environmentId": "environment_1586156073228_0001", + "environmentSpec": { + "name": "my-submarine-env", + "dockerImage" : "continuumio/anaconda3", + "kernelSpec" : { + "name" : "team_default_python_3.7_updated", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0"] + } + } + } +} +``` +> dockerImage, "name" (of kernelSpec), "channels", "dependencies" etc can be updated using this API. +"name" of EnvironmentSpec is not supported. + +### Delete environment +`GET /api/v1/environment/{name}` + +**Example Request:** +```sh +curl -X DELETE http://127.0.0.1:8080/api/v1/environment/my-submarine-env +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "environmentId": "environment_1586156073228_0001", + "environmentSpec": { + "name": "my-submarine-env", + "dockerImage" : "continuumio/anaconda3", + "kernelSpec" : { + "name" : "team_default_python_3.7_updated", + "channels" : ["defaults"], + "dependencies" : + ["_ipyw_jlab_nb_ext_conf=0.1.0=py37_0", + "alabaster=0.7.12=py37_0"] + } + } + } +} +``` \ No newline at end of file diff --git a/docs/userdocs/k8s/api/experiment-template.md b/docs/userdocs/k8s/api/experiment-template.md new file mode 100644 index 0000000000000000000000000000000000000000..055adb12ffd0ab30e7ace74ab9accbc4bb5cc3ce --- /dev/null +++ b/docs/userdocs/k8s/api/experiment-template.md @@ -0,0 +1,235 @@ + + +# Experiment Template API Reference + +> Note: The Experiment API is in the alpha stage which is subjected to incompatible changes in +> future releases. + + +Developers can register a parameterized experiment as an experiment template, +For example, if the developer wants to change the following "--learning_rate=0.1" to parameters. +```json +"experimentSpec": { + "meta": { + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.1 --batch_size=150" + }, + "...": "..." +} +``` + +They can use two curly braces as placeholders, the template format will be as +```json +"experimentSpec": { + "meta": { + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate={{learning_rate}} --batch_size=150" + }, + "...": "..." +} +``` + +The template parameters format will be as +```json +{ + "name": "learning_rate", + "value": 0.1, + "required": true, + "description": "This is learning_rate of training." +} +``` +name: placeholder name +value; default value +required: Indicates whether the user must enter parameters, when required is true, value can be null +description: Introduction of this parameter + +Users can use existing experiment templates and adjust the default value to create experiments. +After the user submits the experiment template, the submarine server finds the corresponding template based on the name. And the template handler converts input parameters to an actual experiment, such as a distributed TF experiment. + +The "replicas", "cpu", "memory" of resources will be automatically parameterized, so developers do not need to add them. +For example, if there are "Ps" and "Worker" under spec, the following parameters will be automatically appended. +``` +spec.Ps.replicas +spec.Ps.resourceMap.cpu +spec.Ps.resourceMap.memory +spec.Worker.replicas +spec.Worker.resourceMap.cpu +spec.Worker.resourceMap.memory +``` + + +## Create experiment template +`POST /api/v1/template` + +**Example Request** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "name": "my-tf-mnist-template", + "author": "author", + "description": "This is a template to run tf-mnist", + "parameters": [{ + "name": "learning_rate", + "value": 0.1, + "required": true, + "description": "This is learning_rate of training." + }, + { + "name": "batch_size", + "value": 150, + "required": true, + "description": "This is batch_size of training." + }, + { + "name": "experiment_name", + "value": "tf-mnist1", + "required": true, + "description": "the name of experiment." + } + ], + "experimentSpec": { + "meta": { + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate={{learning_rate}} --batch_size={{batch_size}}", + "name": "{{experiment_name}}", + "envVars": { + "ENV1": "ENV1" + }, + "framework": "TensorFlow", + "namespace": "default" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + } + } +} +' http://127.0.0.1:8080/api/v1/template +``` + + +### List experiment template +`GET /api/v1/template` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/template +``` + +### Get experiment template +`GET /api/v1/template/{name}` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/template/my-tf-mnist-template +``` + + +### Patch template +`PATCH /api/v1/template/{name}` +```sh +curl -X PATCH -H "Content-Type: application/json" -d ' +{ + "name": "my-tf-mnist-template", + "author": "author-new", + "description": "This is a template to run tf-mnist", + "parameters": [{ + "name": "learning_rate", + "value": 0.1, + "required": true, + "description": "This is learning_rate of training." + }, + { + "name": "batch_size", + "value": 150, + "required": true, + "description": "This is batch_size of training." + }, + { + "name": "experiment_name", + "value": "tf-mnist1", + "required": true, + "description": "the name of experiment." + } + ], + "experimentSpec": { + "meta": { + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate={{learning_rate}} --batch_size={{batch_size}}", + "name": "{{experiment_name}}", + "envVars": { + "ENV1": "ENV1" + }, + "framework": "TensorFlow", + "namespace": "default" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + } + } +} +' http://127.0.0.1:8080/api/v1/template/my-tf-mnist-template +``` + +> "description", "parameters", "experimentSpec", "author" etc can be updated using this API. +"name" of experiment template is not supported. + + + +### Delete template +`GET /api/v1/template/{name}` + +**Example Request:** +```sh +curl -X DELETE http://127.0.0.1:8080/api/v1/template/my-tf-mnist-template +``` + + +### Use template to create a experiment +`POST /api/v1/experiment/{template_name}` + +**Example Request:** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "name": "tf-mnist", + "params": { + "learning_rate":"0.01", + "batch_size":"150", + "experiment_name":"newexperiment1" + } +} +' http://127.0.0.1:8080/api/v1/experiment/my-tf-mnist-template +``` diff --git a/docs/userdocs/k8s/api/experiment.md b/docs/userdocs/k8s/api/experiment.md new file mode 100644 index 0000000000000000000000000000000000000000..703323eb62523e045c5319f8d6ecb7b1cc5b4afa --- /dev/null +++ b/docs/userdocs/k8s/api/experiment.md @@ -0,0 +1,483 @@ + + +# Experiment API Reference + +> Note: The Experiment API is in the alpha stage which is subjected to incompatible changes in +> future releases. + +## Create Experiment (Using Anonymous/Embedded Environment) +`POST /api/v1/experiment` + +**Example Request** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + } + } +} +' http://127.0.0.1:8080/api/v1/experiment +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "experimentId": "experiment_1586156073228_0001", + "name": "tf-mnist-json", + "uid": "28e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:59:29.000+08:00", + "spec": { + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + } + } + } + } +} +``` + +## Create Experiment (Using Pre-defined/Stored Environment) +`POST /api/v1/experiment` + +**Example Request** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "name": "my-submarine-env" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + } + } +} +' http://127.0.0.1:8080/api/v1/experiment +``` +Above example assume environment "my-submarine-env" already exists in Submarine. Please refer Environment API Reference doc to Create/Update/Delete/List Environment REST API's + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "experimentId": "experiment_1586156073228_0001", + "name": "tf-mnist-json", + "uid": "28e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:59:29.000+08:00", + "spec": { + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "name": "my-submarine-env" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + } + } + } + } +} +``` + +### List experiment +`GET /api/v1/experiment` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/experiment +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": [ + { + "experimentId": "experiment_1592057447228_0001", + "name": "tf-mnist-json", + "uid": "28e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:59:29.000+08:00", + "spec": { + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + } + } + } + }, + { + "experimentId": "experiment_1592057447228_0002", + "name": "mnist", + "uid": "38e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:19:29.000+08:00", + "spec": { + "meta": { + "name": "pytorch-mnist-json", + "namespace": "default", + "framework": "PyTorch", + "cmd": "python /var/mnist.py --backend gloo", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:pytorch-dist-mnist-1.0" + }, + "spec": { + "Master": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + } + } + } + ] +} +``` + +### Get experiment +`GET /api/v1/experiment/{id}` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/experiment/experiment_1592057447228_0001 +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "experimentId": "experiment_1592057447228_0001", + "name": "tf-mnist-json", + "uid": "28e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:59:29.000+08:00", + "spec": { + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + } + } + } + } +} +``` + +### Patch experiment +`PATCH /api/v1/experiment/{id}` + +**Example Request:** +```sh +curl -X PATCH -H "Content-Type: application/json" -d ' +{ + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 2, + "resources": "cpu=1,memory=512M" + } + } +} +' http://127.0.0.1:8080/api/v1/experiment/experiment_1592057447228_0001 +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "success": true, + "result": { + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 2, + "resources": "cpu=1,memory=512M" + } + } + } +} +``` + +### Delete experiment +`GET /api/v1/experiment/{id}` + +**Example Request:** +```sh +curl -X DELETE http://127.0.0.1:8080/api/v1/experiment/experiment_1592057447228_0001 +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "experimentId": "experiment_1586156073228_0001", + "name": "tf-mnist-json", + "uid": "28e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:59:29.000+08:00", + "spec": { + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=512M" + }, + "Worker": { + "replicas": 2, + "resources": "cpu=1,memory=512M" + } + } + } + } +} +``` + +### List experiment Log +`GET /api/v1/experiment/logs` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/experiment/logs +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "success": null, + "message": null, + "result": [ + { + "experimentId": "experiment_1589199154923_0001", + "logContent": [ + { + "podName": "mnist-worker-0", + "podLog": null + } + ] + }, + { + "experimentId": "experiment_1589199154923_0002", + "logContent": [ + { + "podName": "pytorch-dist-mnist-gloo-master-0", + "podLog": null + }, + { + "podName": "pytorch-dist-mnist-gloo-worker-0", + "podLog": null + } + ] + } + ], + "attributes": {} +} +``` + +### Get experiment Log +`GET /api/v1/experiment/logs/{id}` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/experiment/logs/experiment_1589199154923_0002 +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "success": null, + "message": null, + "result": { + "experimentId": "experiment_1589199154923_0002", + "logContent": [ + { + "podName": "pytorch-dist-mnist-gloo-master-0", + "podLog": "Using distributed PyTorch with gloo backend\nDownloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\nDownloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\nDownloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\nDownloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\nProcessing...\nDone!\nTrain Epoch: 1 [0/60000 (0%)]\tloss=2.3000\nTrain Epoch: 1 [640/60000 (1%)]\tloss=2.2135\nTrain Epoch: 1 [1280/60000 (2%)]\tloss=2.1704\nTrain Epoch: 1 [1920/60000 (3%)]\tloss=2.0766\nTrain Epoch: 1 [2560/60000 (4%)]\tloss=1.8679\nTrain Epoch: 1 [3200/60000 (5%)]\tloss=1.4135\nTrain Epoch: 1 [3840/60000 (6%)]\tloss=1.0003\nTrain Epoch: 1 [4480/60000 (7%)]\tloss=0.7762\nTrain Epoch: 1 [5120/60000 (9%)]\tloss=0.4598\nTrain Epoch: 1 [5760/60000 (10%)]\tloss=0.4860\nTrain Epoch: 1 [6400/60000 (11%)]\tloss=0.4389\nTrain Epoch: 1 [7040/60000 (12%)]\tloss=0.4084\nTrain Epoch: 1 [7680/60000 (13%)]\tloss=0.4602\nTrain Epoch: 1 [8320/60000 (14%)]\tloss=0.4289\nTrain Epoch: 1 [8960/60000 (15%)]\tloss=0.3990\nTrain Epoch: 1 [9600/60000 (16%)]\tloss=0.3852\n" + }, + { + "podName": "pytorch-dist-mnist-gloo-worker-0", + "podLog": "Using distributed PyTorch with gloo backend\nDownloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\nDownloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\nDownloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\nDownloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\nProcessing...\nDone!\nTrain Epoch: 1 [0/60000 (0%)]\tloss=2.3000\nTrain Epoch: 1 [640/60000 (1%)]\tloss=2.2135\nTrain Epoch: 1 [1280/60000 (2%)]\tloss=2.1704\nTrain Epoch: 1 [1920/60000 (3%)]\tloss=2.0766\nTrain Epoch: 1 [2560/60000 (4%)]\tloss=1.8679\nTrain Epoch: 1 [3200/60000 (5%)]\tloss=1.4135\nTrain Epoch: 1 [3840/60000 (6%)]\tloss=1.0003\nTrain Epoch: 1 [4480/60000 (7%)]\tloss=0.7762\nTrain Epoch: 1 [5120/60000 (9%)]\tloss=0.4598\nTrain Epoch: 1 [5760/60000 (10%)]\tloss=0.4860\nTrain Epoch: 1 [6400/60000 (11%)]\tloss=0.4389\nTrain Epoch: 1 [7040/60000 (12%)]\tloss=0.4084\nTrain Epoch: 1 [7680/60000 (13%)]\tloss=0.4602\nTrain Epoch: 1 [8320/60000 (14%)]\tloss=0.4289\nTrain Epoch: 1 [8960/60000 (15%)]\tloss=0.3990\nTrain Epoch: 1 [9600/60000 (16%)]\tloss=0.3852\n" + } + ] + }, + "attributes": {} +} +``` diff --git a/docs/userdocs/k8s/api/notebook.md b/docs/userdocs/k8s/api/notebook.md new file mode 100644 index 0000000000000000000000000000000000000000..ca40e48abdd9d28b7da5b9c1968daff267c2bc54 --- /dev/null +++ b/docs/userdocs/k8s/api/notebook.md @@ -0,0 +1,265 @@ + + +# Notebook API Reference + +> Note: The Notebook API is in the alpha stage which is subjected to incompatible changes in future releases. + +## Create a notebook instance +`POST /api/v1/notebook` + +**Example Request** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "meta": { + "name": "test-nb", + "namespace": "default", + "ownerId": "e9ca23d68d884d4ebb19d07889727dae" + }, + "environment": { + "name": "notebook-env" + }, + "spec": { + "envVars": { + "TEST_ENV": "test" + }, + "resources": "cpu=1,memory=1.0Gi" + } +} +' http://127.0.0.1:8080/api/v1/notebook +``` + +**Example Response:** +```json +{ + "status":"OK", + "code":200, + "success":true, + "message":"Create a notebook instance", + "result":{ + "notebookId":"notebook_1597931805405_0001", + "name":"test-nb", + "uid":"5a94c01d-6a92-4222-bc66-c610c277546d", + "url":"/notebook/default/test-nb/", + "status":"Created", + "createdTime":"2020-08-20T21:58:27.000+08:00", + "deletedTime":null, + "spec":{ + "meta":{ + "name":"test-nb", + "namespace":"default", + "ownerId":"e9ca23d68d884d4ebb19d07889727dae" + }, + "environment":{ + "name":"notebook-env", + "dockerImage":"apache/submarine:jupyter-notebook-0.5.0", + "kernelSpec":{ + "name": "team_default_python_3.7", + "channels": [ + "defaults" + ], + "dependencies": [ + "" + ] + }, + "description":null, + "image":null + }, + "spec":{ + "envVars":{ + "TEST_ENV":"test" + }, + "resources":"cpu=1,memory=1.0Gi" + } + } + }, + "attributes":{} +} +``` + +## List notebook instances which belong to user +`GET /api/v1/notebook` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/notebook?id={user_id} +``` + +**Example Response:** +```json +{ + "status":"OK", + "code":200, + "success":true, + "message":"List all notebook instances", + "result":[ + { + "notebookId":"notebook_1597931805405_0001", + "name":"test-nb", + "uid":"5a94c01d-6a92-4222-bc66-c610c277546d", + "url":"/notebook/default/test-nb/", + "status":"Created", + "createdTime":"2020-08-20T21:58:27.000+08:00", + "deletedTime":null, + "spec":{ + "meta":{ + "name":"test-nb", + "namespace":"default", + "ownerId":"e9ca23d68d884d4ebb19d07889727dae" + }, + "environment":{ + "name":"notebook-env", + "dockerImage":"apache/submarine:jupyter-notebook-0.5.0", + "kernelSpec":{ + "name": "team_default_python_3.7", + "channels": [ + "defaults" + ], + "dependencies": [ + "" + ] + }, + "description":null, + "image":null + }, + "spec":{ + "envVars":{ + "TEST_ENV":"test" + }, + "resources":"cpu=1,memory=1.0Gi" + } + } + } + ], + "attributes":{} +} +``` + +## Get the notebook instance +`GET /api/v1/notebook/{id}` + +**Example Request:** +```sh +curl -X GET http://127.0.0.1:8080/api/v1/notebook/{id} +``` + +**Example Response:** +```json +{ + "status":"OK", + "code":200, + "success":true, + "message":"Get the notebook instance", + "result":{ + "notebookId":"notebook_1597931805405_0001", + "name":"test-nb", + "uid":"5a94c01d-6a92-4222-bc66-c610c277546d", + "url":"/notebook/default/test-nb/", + "status":"Created", + "createdTime":"2020-08-20T21:58:27.000+08:00", + "deletedTime":null, + "spec":{ + "meta":{ + "name":"test-nb", + "namespace":"default", + "ownerId":"e9ca23d68d884d4ebb19d07889727dae" + }, + "environment":{ + "name":"notebook-env", + "dockerImage":"apache/submarine:jupyter-notebook-0.5.0", + "kernelSpec":{ + "name": "team_default_python_3.7", + "channels": [ + "defaults" + ], + "dependencies": [ + "" + ] + }, + "description":null, + "image":null + }, + "spec":{ + "envVars":{ + "TEST_ENV":"test" + }, + "resources":"cpu=1,memory=1.0Gi" + } + } + }, + "attributes":{} +} +``` + +## Delete the notebook instance +`DELETE /api/v1/notebook/{id}` + +**Example Request:** +```sh +curl -X DELETE http://127.0.0.1:8080/api/v1/notebook/{id} +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "success": true, + "message": "Delete the notebook instance", + "result": { + "notebookId": "notebook_1597931805405_0001", + "name": "test-nb", + "uid": "5a94c01d-6a92-4222-bc66-c610c277546d", + "url": "/notebook/default/test-nb/", + "status": "Deleted", + "createdTime": "2020-08-22T14:03:19.000+08:00", + "deletedTime": "2020-08-22T14:46:28+0800", + "spec": { + "meta": { + "name": "test-nb", + "namespace": "default", + "ownerId":"e9ca23d68d884d4ebb19d07889727dae" + }, + "environment": { + "name": "notebook-env", + "dockerImage": "apache/submarine:jupyter-notebook-0.5.0", + "kernelSpec": { + "name": "team_default_python_3.7", + "channels": [ + "defaults" + ], + "dependencies": [ + "" + ] + }, + "description": null, + "image": null + }, + "spec": { + "envVars": { + "TEST_ENV": "test" + }, + "resources": "cpu=1,memory=1.0Gi" + } + } + }, + "attributes": {} +} +``` diff --git a/docs/userdocs/k8s/helm.md b/docs/userdocs/k8s/helm.md new file mode 100644 index 0000000000000000000000000000000000000000..99d68e23497a8489507b4a4b1b3e87d248d1bf14 --- /dev/null +++ b/docs/userdocs/k8s/helm.md @@ -0,0 +1,120 @@ + + + +# Deploy Submarine On K8s + +## Deploy Submarine Using Helm Chart (Recommended) + +Submarine's Helm Chart will deploy Submarine Server, TF/PyTorch Operator, Notebook controller +and Traefik. We use the TF/PyTorch operator to run tf/pytorch job, the notebook controller to +manage jupyter notebook and Traefik as reverse-proxy. + + +### Install Helm + +Helm v3 is minimum requirement. +See here for installation: https://helm.sh/docs/intro/install/ + +### Install Submarine + +The Submarine helm charts is released with the source code for now. +Please go to `http://submarine.apache.org/download.html` to download + +- Install Helm charts from source code +```bash +cd /submarine +helm install submarine ./helm-charts/submarine +``` +This will install submarine in the "default" namespace. +The images are from Docker hub `apache/submarine`. See `./helm-charts/submarine/values.yaml` for more details + +If we'd like use a different namespace like "submarine" +```bash +kubectl create namespace submarine +helm install submarine ./helm-charts/submarine -n submarine +``` + +> Note that if you encounter below issue when installation: +```bash +Error: rendered manifests contain a resource that already exists. +Unable to continue with install: existing resource conflict: namespace: , name: podgroups.scheduling.incubator.k8s.io, existing_kind: apiextensions.k8s.io/v1beta1, Kind=CustomResourceDefinition, new_kind: apiextensions.k8s.io/v1beta1, Kind=CustomResourceDefinition +``` +It might be caused by the previous installed submarine charts. Fix it by running: +```bash +kubectl delete crd/tfjobs.kubeflow.org && kubectl delete crd/podgroups.scheduling.incubator.k8s.io && kubectl delete crd/pytorchjobs.kubeflow.org +``` + +- Verify installation + +Once you got it installed, check with below commands and you should see similar outputs: +```bash +kubectl get pods +``` + +```bash +NAME READY STATUS RESTARTS AGE +notebook-controller-deployment-5db8b6cbf7-k65jm 1/1 Running 0 5s +pytorch-operator-7ff5d96d59-gx7f5 1/1 Running 0 5s +submarine-database-8d95d74f7-ntvqp 1/1 Running 0 5s +submarine-server-b6cd4787b-7bvr7 1/1 Running 0 5s +submarine-traefik-9bb6f8577-66sx6 1/1 Running 0 5s +tf-job-operator-7844656dd-lfgmd 1/1 Running 0 5s +``` + +### Access to Submarine Server +Submarine server by default expose 8080 port within K8s cluster. After Submarine v0.5 +uses Traefik as reverse-proxy by default. If you don't want to +use Traefik, you can modify below value to ***false*** in `./helm-charts/submarine/values.yaml`. +```yaml +# Use Traefik by default +traefik: + enabled: true +``` + +To access the server from outside of the cluster, we use Traefik ingress controller and +NodePort for external access.\ +Please refer to `./helm-charts/submarine/charts/traefik/values.yaml` and [Traefik docs](https://docs.traefik.io/) +for more details if you want to customize the default value for Traefik. + +``` +# Use nodePort and Traefik ingress controller by default. +# To access the submarine server, open the following URL in your browser. +http://127.0.0.1:32080 +``` + +*Notice:* +If you use `kind` to run local Kubernetes cluster, +please refer to this [docs](https://kind.sigs.k8s.io/docs/user/configuration/#extra-port-mappings) +and set the configuration "extraPortMappings" when creating the k8s cluster. + +``` +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 32080 + hostPort: [the port you want to access] +``` + +### Uninstall Submarine +```bash +helm delete submarine +``` diff --git a/docs/userdocs/k8s/kind.md b/docs/userdocs/k8s/kind.md new file mode 100644 index 0000000000000000000000000000000000000000..cb472c4b356ccdfef23e4d376754bff5f61117d2 --- /dev/null +++ b/docs/userdocs/k8s/kind.md @@ -0,0 +1,61 @@ + + +# Kind Guide + +## Create K8s cluster +We recommend using [`kind`](https://kind.sigs.k8s.io/) to setup a Kubernetes cluster on a local machine. + +Running the following command: +``` +kind create cluster --image kindest/node:v1.15.6 --name submarine +kubectl create namespace submarine +``` + +## Kubernetes Dashboard (optional) + +### Deploy +To deploy Dashboard, execute following command: +``` +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta8/aio/deploy/recommended.yaml +``` + +### Create RBAC +Ensure to grant the cluster access permission of dashboard, run the following command: +``` +kubectl create serviceaccount dashboard-admin-sa +kubectl create clusterrolebinding dashboard-admin-sa --clusterrole=cluster-admin --serviceaccount=default:dashboard-admin-sa +``` + +### Get access token (optional) +If you want to use the token to login the dashboard, run the following command to get key: +``` +kubectl get secrets +# select the right dashboard-admin-sa-token to describe the secret +kubectl describe secret dashboard-admin-sa-token-6nhkx +``` + +### Start dashboard service +To start the dashboard service, we can run the following command: +``` +kubectl proxy +``` + +Now access Dashboard at: +> http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/ diff --git a/docs/userdocs/k8s/notebook.md b/docs/userdocs/k8s/notebook.md new file mode 100644 index 0000000000000000000000000000000000000000..4c1d1f2016875625bf4a1f3e95d9cd1e5c536330 --- /dev/null +++ b/docs/userdocs/k8s/notebook.md @@ -0,0 +1,101 @@ + + +# Notebooks Guide + +This guide describes how to use Jupyter notebook in Submarine to launch +and manage Jupyter notebooks. + +## Working with notebooks + +We recommend using Web UI to manage notebooks. + +### Notebooks Web UI + +Notebooks can be started from the Web UI. You can click the “Notebook” tab in the \ +left-hand panel to manage your notebooks. + +#### + +To create a new notebook server, click “New Notebook”. You should see a form for entering \ +details of your new notebook server. + +- Notebook Name : Name of the notebook server. It should follow the rules below. + 1. Contain at most 63 characters. + 2. Contain only lowercase alphanumeric characters or '-'. + 3. Start with an alphabetic character. + 4. End with an alphanumeric character. +- Environment : It defines a set of libraries and docker image. +- CPU and Memory +- GPU (optional) +- EnvVar (optional) : Injects environment variables into the notebook. + +**If you’re not sure which environment you need, please choose the environment “notebook-env” \ +for the new notebook.** + +#### + +You should see your new notebook server. Click the name of your notebook server to connect to it. + +#### + +## Experiment with your notebook + +The environment “notebook-env” includes Submarine Python SDK which can talk to Submarine Server to \ +create experiments, as the example below: + +```python +from __future__ import print_function +import submarine +from submarine.experiment.models.environment_spec import EnvironmentSpec +from submarine.experiment.models.experiment_spec import ExperimentSpec +from submarine.experiment.models.experiment_task_spec import ExperimentTaskSpec +from submarine.experiment.models.experiment_meta import ExperimentMeta +from submarine.experiment.models.code_spec import CodeSpec + +# Create Submarine Client +submarine_client = submarine.ExperimentClient() + +# Define TensorFlow experiment spec +environment = EnvironmentSpec(image='apache/submarine:tf-dist-mnist-test-1.0') +experiment_meta = ExperimentMeta(name='mnist-dist', + namespace='default', + framework='Tensorflow', + cmd='python /var/tf_dist_mnist/dist_mnist.py --train_steps=100', + env_vars={'ENV1': 'ENV1'}) + +worker_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M', + replicas=1) +ps_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M', + replicas=1) +code_spec = CodeSpec(sync_mode='git', url='https://github.com/apache/submarine.git') + +experiment_spec = ExperimentSpec(meta=experiment_meta, + environment=environment, + code=code_spec, + spec={'Ps' : ps_spec,'Worker': worker_spec}) + +# Create experiment +experiment = submarine_client.create_experiment(experiment_spec=experiment_spec) + +``` + +You can create a new notebook, paste the above code and run it. Or, you can find the notebook `submarine_experiment_sdk.ipynb` inside the launched notebook session. You can open it, try it out. + +After experiment submitted to Submarine server, you can find the experiment jobs on the UI. diff --git a/docs/userdocs/k8s/run-experiment-python-sdk.md b/docs/userdocs/k8s/run-experiment-python-sdk.md new file mode 100644 index 0000000000000000000000000000000000000000..c28fdc67ca8218195b3240e074e14aa0c7e32e50 --- /dev/null +++ b/docs/userdocs/k8s/run-experiment-python-sdk.md @@ -0,0 +1,64 @@ + + +Submarine Python SDK can runs on any machine and it will talk to Submarine Server via REST API. So you can install Submarine Python SDK on your laptop, a gateway machine, your favorite IDE (like PyCharm/Jupyter, etc.). + +#### Prepare Python Environment to run Submarine SDK + +First of all + +Submarine SDK requires Python3.7+. +It's better to use a new Python environment created by `Anoconda` or Python `virtualenv` to try this to avoid trouble to existing Python environment. +A sample Python virtual env can be setup like this: + +```bash +wget https://files.pythonhosted.org/packages/33/bc/fa0b5347139cd9564f0d44ebd2b147ac97c36b2403943dbee8a25fd74012/virtualenv-16.0.0.tar.gz +tar xf virtualenv-16.0.0.tar.gz + +# Make sure to install using Python 3 +python3 virtualenv-16.0.0/virtualenv.py venv +. venv/bin/activate +``` + +#### Install Submarine SDK + +**Install SDK from pypi.org (recommended)** + +Starting from 0.4.0, Submarine provides Python SDK. Please change it to a proper version needed. + +```bash +pip install submarine-sdk== +``` + +**Install SDK from source code** + +Please first clone code from github or go to `http://submarine.apache.org/download.html` to download released source code. + +```bash +git clone https://github.com/apache/submarine.git +git checkout +cd submarine/submarine-sdk/pysubmarine +pip install . +``` + +#### Run with Submarine Python SDK + +Assuming you've installed submarine on K8s and forward the service to localhost, now you can open a Python shell, Jupyter notebook or any tools with Submarine SDK installed. + +Follow [SDK experiment example](../../../submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb) to try the SDK. \ No newline at end of file diff --git a/docs/userdocs/k8s/run-experiment-template-rest.md b/docs/userdocs/k8s/run-experiment-template-rest.md new file mode 100644 index 0000000000000000000000000000000000000000..183f760f4344b472566994625c008595e02817e5 --- /dev/null +++ b/docs/userdocs/k8s/run-experiment-template-rest.md @@ -0,0 +1,54 @@ + + +# Experiment Template REST API Reference + +## Experiment Template Spec +The experiment is represented in [JSON](https://www.json.org) or [YAML](https://yaml.org) format. + + +### Use existing experiment template to create a experiment +`POST /api/v1/environment/{name}` + +**Example Request:** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "name": "tf-mnist", + "params": { + "learning_rate":"0.01", + "batch_size":"150", + "experiment_name":"newexperiment1" + } +} +' http://127.0.0.1:8080/api/v1/experiment/my-tf-mnist-template +``` + +**Example Request:** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "params": { + "experiment_name":"new-pytorch-mnist" + } +} +' http://127.0.0.1:8080/api/v1/experiment/pytorch-mnist +``` + +Register experiment template and more info see [Experiment Template API Reference](api/experiment-template.md). diff --git a/docs/userdocs/k8s/run-pytorch-experiment-rest.md b/docs/userdocs/k8s/run-pytorch-experiment-rest.md new file mode 100644 index 0000000000000000000000000000000000000000..3fcac79607c20e113ea43853cdd33e896a744954 --- /dev/null +++ b/docs/userdocs/k8s/run-pytorch-experiment-rest.md @@ -0,0 +1,146 @@ + + +# Run PyTorch Experiment Guide (REST) + +## Experiment Spec +The experiment is represented in [JSON](https://www.json.org) or [YAML](https://yaml.org) format. + +**YAML Format:** +```yaml +meta: + name: pytorch-mnist-yaml + namespace: default + framework: PyTorch + cmd: python /var/mnist.py --backend gloo + envVars: + ENV_1: ENV1 +environment: + image: apache/submarine:pytorch-dist-mnist-1.0 +spec: + Master: + replicas: 1 + resources: cpu=1,memory=1024M + Worker: + replicas: 1 + resources: cpu=1,memory=1024M +``` + +**JSON Format:** +```json +{ + "meta": { + "name": "pytorch-mnist-json", + "namespace": "default", + "framework": "PyTorch", + "cmd": "python /var/mnist.py --backend gloo", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:pytorch-dist-mnist-1.0" + }, + "spec": { + "Master": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + } +} + +``` + +## Create Experiment by REST API +`POST /api/v1/experiment` + +**Example Request** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "meta": { + "name": "pytorch-mnist-json", + "namespace": "default", + "framework": "PyTorch", + "cmd": "python /var/mnist.py --backend gloo", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:pytorch-dist-mnist-1.0" + }, + "spec": { + "Master": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + } +} +' http://127.0.0.1:8080/api/v1/experiment +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "experimentId": "experiment_1592057447228_0002", + "name": "mnist", + "uid": "38e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:19:29.000+08:00", + "spec": { + "meta": { + "name": "pytorch-mnist-json", + "namespace": "default", + "framework": "PyTorch", + "cmd": "python /var/mnist.py --backend gloo", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:pytorch-dist-mnist-1.0" + }, + "spec": { + "Master": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + } + } + } +} +``` + +More info see [Experiment API Reference](api/experiment.md). diff --git a/docs/userdocs/k8s/run-tensorflow-experiment-rest.md b/docs/userdocs/k8s/run-tensorflow-experiment-rest.md new file mode 100644 index 0000000000000000000000000000000000000000..5145dc066001ab787c1b7f1df643faabee8c11d1 --- /dev/null +++ b/docs/userdocs/k8s/run-tensorflow-experiment-rest.md @@ -0,0 +1,145 @@ + + +# Run TensorFlow Experiment Guide (REST) + +## Experiment Spec +The experiment is represented in [JSON](https://www.json.org) or [YAML](https://yaml.org) format. + +**YAML Format:** +```yaml +meta: + name: "tf-mnist-yaml" + namespace: "default" + framework: "TensorFlow" + cmd: "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150" + envVars: + ENV_1: "ENV1" +environment: + image: "apache/submarine:tf-mnist-with-summaries-1.0" +spec: + Ps: + replicas: 1 + resources: "cpu=1,memory=1024M" + Worker: + replicas: 1 + resources: "cpu=1,memory=1024M" +``` + +**JSON Format:** +```json +{ + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + } +} +``` + +## Create Experiment by REST API +`POST /api/v1/experiment` + +**Example Request** +```sh +curl -X POST -H "Content-Type: application/json" -d ' +{ + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + } +} +' http://127.0.0.1:8080/api/v1/experiment +``` + +**Example Response:** +```json +{ + "status": "OK", + "code": 200, + "result": { + "experimentId": "experiment_1592057447228_0001", + "name": "tf-mnist-json", + "uid": "28e39dcd-77d4-11ea-8dbb-0242ac110003", + "status": "Accepted", + "acceptedTime": "2020-06-13T22:59:29.000+08:00", + "spec": { + "meta": { + "name": "tf-mnist-json", + "namespace": "default", + "framework": "TensorFlow", + "cmd": "python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150", + "envVars": { + "ENV_1": "ENV1" + } + }, + "environment": { + "image": "apache/submarine:tf-mnist-with-summaries-1.0" + }, + "spec": { + "Ps": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + }, + "Worker": { + "replicas": 1, + "resources": "cpu=1,memory=1024M" + } + } + } + } +} +``` + +More info see [Experiment API Reference](api/experiment.md). diff --git a/docs/userdocs/k8s/run-tensorflow-experiment-ui.md b/docs/userdocs/k8s/run-tensorflow-experiment-ui.md new file mode 100644 index 0000000000000000000000000000000000000000..f68a878ffbc4b6e3ab6f5abfed238bfba15a89fe --- /dev/null +++ b/docs/userdocs/k8s/run-tensorflow-experiment-ui.md @@ -0,0 +1,56 @@ + + +# Run Tensorflow Experiment on Submarine UI + +## Steps to run Tensorflow Experiment + +- Click `+ New Experiment` on the "Experiment" page. + +- Click `Define your experiment` + +- Put a name to experiment, like "minst-example" + +- Command: `python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150` + +- Image you can put; `apache/submarine:tf-mnist-with-summaries-1.0` + +- Click `Next Step` + +- Choose `Distributed Tensorflow` + +- Click `Add new spec` twice to add two new specs (roles) + +- One is Worker, another one is PS, leave rest of the parameters unchanged + +- Click next step, you can review your parameters before submitting the job: + + + + It should look like below: + + | Name | mnist-example-111 | | | + | --------------------- | ------------------------------------------------------------ | ---- | ---- | + | Command | python /var/tf_mnist/mnist_with_summaries.py --log_dir=/train/log --learning_rate=0.01 --batch_size=150 | | | + | Image | apache/submarine:tf-mnist-with-summaries-1.0 | | | + | Environment Variables | | | | + | Ps | cpu=1,nvidia.com/gpu=0,memory=1024M | | | + | Worker | cpu=1,nvidia.com/gpu=0,memory=1024M | | | + +- Click `Submit` it will be submitted, you can see the new example running in the `Experiment` list, you can get logs, etc. directly from the UI \ No newline at end of file diff --git a/docs/userdocs/k8s/tensorflow.md b/docs/userdocs/k8s/tensorflow.md new file mode 100644 index 0000000000000000000000000000000000000000..e3cd2bac0aa88e122d859920fadd116bdbe56409 --- /dev/null +++ b/docs/userdocs/k8s/tensorflow.md @@ -0,0 +1,54 @@ + + +# Deploy Tensorflow Operator on Kubernetes + +## TFJob +We support Tensorflow job on kubernetes by using the tf-operator as a runtime. For more info about tf-operator see [here](https://github.com/kubeflow/tf-operator). + +### Deploy tf-operator +> If you don't have the `submarine` namespace on your K8s cluster, you should create it first. Run command: `kubectl create namespace submarine` + +Running the follow commands: +``` +kubectl apply -f ./dev-support/k8s/tfjob/crd.yaml +kubectl kustomize ./dev-support/k8s/tfjob/operator | kubectl apply -f - +``` + +> Since K8s 1.14, Kubectl also supports the management of Kubernetes objects using a kustomization file. For more info see [kustomization](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/) + +Default namespace is `submarine`, if you want to modify the namespace, please modify `./dev-support/k8s/tfjob/operator/kustomization.yaml`, such as modify `${NAMESPACE}` as below: +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ${NAMESPACE} +resources: +- cluster-role-binding.yaml +- cluster-role.yaml +- deployment.yaml +- service-account.yaml +- service.yaml +commonLabels: + kustomize.component: tf-job-operator +images: +- name: apache/submarine + newName: apache/submarine + newTag: tf_operator-v1.1.0-g92389064 +``` + diff --git a/docs/userdocs/yarn/Dockerfiles.md b/docs/userdocs/yarn/Dockerfiles.md new file mode 100644 index 0000000000000000000000000000000000000000..1796d1e2f80ebed7e1ad9223d83334129c6f4774 --- /dev/null +++ b/docs/userdocs/yarn/Dockerfiles.md @@ -0,0 +1,22 @@ + + +# Write Dockerfiles for Submarine + +[How to write Dockerfile for Submarine TensorFlow jobs](WriteDockerfileTF.md) + +[How to write Dockerfile for Submarine PyTorch jobs](WriteDockerfilePT.md) + +[How to write Dockerfile for Submarine MXNet jobs](WriteDockerfileMX.md) diff --git a/docs/userdocs/yarn/README.md b/docs/userdocs/yarn/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e1fd423ca04d470eb6c08b0e24f97e333b470cc --- /dev/null +++ b/docs/userdocs/yarn/README.md @@ -0,0 +1,37 @@ + + +# Running Submarine on YARN + +Submarine for YARN supports TensorFlow, PyTorch and MXNet framework. (Which is leveraging [TonY](https://github.com/linkedin/TonY) created by Linkedin to run deep learning training jobs on YARN. + +Submarine also supports GPU-on-YARN and Docker-on-YARN feature. + +Submarine can run on Hadoop 2.7.3 or later version, if GPU-on-YARN or Docker-on-YARN feature is needed, newer Hadoop version is required, please refer to the next section about what Hadoop version to choose. + +## Hadoop version + +Must: + +- Apache Hadoop version newer than 2.7.3 + +Optional: + +- When you want to use GPU-on-YARN feature with Submarine, please make sure Hadoop is at least 2.10.0+ (or 3.1.0+), and follow [Enable GPU on YARN 2.10.0+](https://hadoop.apache.org/docs/r2.10.0/hadoop-yarn/hadoop-yarn-site/UsingGpus.html) to enable GPU-on-YARN feature. +- When you want to run training jobs with Docker container, please make sure Hadoop is at least 2.8.2, and follow [Enable Docker on YARN 2.8.2+](https://hadoop.apache.org/docs/r2.8.2/hadoop-yarn/hadoop-yarn-site/DockerContainers.html) to enable Docker-on-YARN feature. + +## Submarine YARN Runtime Guide + +[YARN Runtime Guide](YARNRuntimeGuide.md) talk about how to use Submarine to run jobs on YARN, with Docker / without Docker. diff --git a/docs/userdocs/yarn/TestAndTroubleshooting.md b/docs/userdocs/yarn/TestAndTroubleshooting.md new file mode 100644 index 0000000000000000000000000000000000000000..21018254544906956bf9b50095dbf60d4ac12601 --- /dev/null +++ b/docs/userdocs/yarn/TestAndTroubleshooting.md @@ -0,0 +1,147 @@ + + +#### Test with a tensorflow job + +Distributed-shell + GPU + cgroup + +```bash + ... \ + job run \ + --env DOCKER_JAVA_HOME=/opt/java \ + --env DOCKER_HADOOP_HDFS_HOME=/hadoop-current --name distributed-tf-gpu \ + --env YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK=calico-network \ + --worker_docker_image tf-1.13.1-gpu:0.0.1 \ + --ps_docker_image tf-1.13.1-cpu:0.0.1 \ + --input_path hdfs://${dfs_name_service}/tmp/cifar-10-data \ + --checkpoint_path hdfs://${dfs_name_service}/user/hadoop/tf-distributed-checkpoint \ + --num_ps 0 \ + --ps_resources memory=4G,vcores=2,gpu=0 \ + --ps_launch_cmd "python /test/cifar10_estimator/cifar10_main.py --data-dir=hdfs://${dfs_name_service}/tmp/cifar-10-data --job-dir=hdfs://${dfs_name_service}/tmp/cifar-10-jobdir --num-gpus=0" \ + --worker_resources memory=4G,vcores=2,gpu=1 --verbose \ + --num_workers 1 \ + --worker_launch_cmd "python /test/cifar10_estimator/cifar10_main.py --data-dir=hdfs://${dfs_name_service}/tmp/cifar-10-data --job-dir=hdfs://${dfs_name_service}/tmp/cifar-10-jobdir --train-steps=500 --eval-batch-size=16 --train-batch-size=16 --sync --num-gpus=1" +``` + + + +## Issues: + +### Issue 1: Fail to start nodemanager after system reboot + +``` +2018-09-20 18:54:39,785 ERROR org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor: Failed to bootstrap configured resource subsystems! +org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException: Unexpected: Cannot create yarn cgroup Subsystem:cpu Mount points:/proc/mounts User:yarn Path:/sys/fs/cgroup/cpu,cpuacct/hadoop-yarn + at org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandlerImpl.initializePreMountedCGroupController(CGroupsHandlerImpl.java:425) + at org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandlerImpl.initializeCGroupController(CGroupsHandlerImpl.java:377) + at org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsCpuResourceHandlerImpl.bootstrap(CGroupsCpuResourceHandlerImpl.java:98) + at org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsCpuResourceHandlerImpl.bootstrap(CGroupsCpuResourceHandlerImpl.java:87) + at org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain.bootstrap(ResourceHandlerChain.java:58) + at org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor.init(LinuxContainerExecutor.java:320) + at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:389) + at org.apache.hadoop.service.AbstractService.init(AbstractService.java:164) + at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:929) + at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:997) +2018-09-20 18:54:39,789 INFO org.apache.hadoop.service.AbstractService: Service NodeManager failed in state INITED +``` + +Solution: Grant user yarn the access to `/sys/fs/cgroup/cpu,cpuacct`, which is the subfolder of cgroup mount destination. + +``` +chown :yarn -R /sys/fs/cgroup/cpu,cpuacct +chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct +``` + +If GPUs are used，the access to cgroup devices folder is neede as well + +``` +chown :yarn -R /sys/fs/cgroup/devices +chmod g+rwx -R /sys/fs/cgroup/devices +``` + + +### Issue 2: container-executor permission denied + +``` +2018-09-21 09:36:26,102 WARN org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor: IOException executing command: +java.io.IOException: Cannot run program "/etc/yarn/sbin/Linux-amd64-64/container-executor": error=13, Permission denied + at java.lang.ProcessBuilder.start(ProcessBuilder.java:1048) + at org.apache.hadoop.util.Shell.runCommand(Shell.java:938) + at org.apache.hadoop.util.Shell.run(Shell.java:901) + at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1213) +``` + +Solution: The permission of `/etc/yarn/sbin/Linux-amd64-64/container-executor` should be 6050 + +### Issue 3：How to get docker service log + +Solution: we can get docker log with the following command + +``` +journalctl -u docker +``` + +### Issue 4：docker can't remove containers with errors like `device or resource busy` + +```bash +$ docker rm 0bfafa146431 +Error response from daemon: Unable to remove filesystem for 0bfafa146431771f6024dcb9775ef47f170edb2f1852f71916ba44209ca6120a: remove /app/docker/containers/0bfafa146431771f6024dcb9775ef47f170edb2f152f71916ba44209ca6120a/shm: device or resource busy +``` + +Solution: to find which process leads to a `device or resource busy`, we can add a shell script, named `find-busy-mnt.sh` + +```bash +#!/usr/bin/env bash + +# A simple script to get information about mount points and pids and their +# mount namespaces. + +if [ $# -ne 1 ];then +echo "Usage: $0 " +exit 1 +fi + +ID=$1 + +MOUNTS=`find /proc/*/mounts | xargs grep $ID 2>/dev/null` + +[ -z "$MOUNTS" ] && echo "No pids found" && exit 0 + +printf "PID\tNAME\t\tMNTNS\n" +echo "$MOUNTS" | while read LINE; do +PID=`echo $LINE | cut -d ":" -f1 | cut -d "/" -f3` +# Ignore self and thread-self +if [ "$PID" == "self" ] || [ "$PID" == "thread-self" ]; then + continue +fi +NAME=`ps -q $PID -o comm=` +MNTNS=`readlink /proc/$PID/ns/mnt` +printf "%s\t%s\t\t%s\n" "$PID" "$NAME" "$MNTNS" +done +``` + +Kill the process by pid, which is found by the script + +```bash +$ chmod +x find-busy-mnt.sh +./find-busy-mnt.sh 0bfafa146431771f6024dcb9775ef47f170edb2f152f71916ba44209ca6120a +# PID NAME MNTNS +# 5007 ntpd mnt:[4026533598] +$ kill -9 5007 +``` + +### Issue 5：Yarn failed to start containers + +if the number of GPUs required by applications is larger than the number of GPUs in the cluster, there would be some containers can't be created. diff --git a/docs/userdocs/yarn/WriteDockerfileMX.md b/docs/userdocs/yarn/WriteDockerfileMX.md new file mode 100644 index 0000000000000000000000000000000000000000..dccf76a7fad4f10b84cd2d78988d48b68a48d48b --- /dev/null +++ b/docs/userdocs/yarn/WriteDockerfileMX.md @@ -0,0 +1,88 @@ + + +# Creating Docker Images for Running MXNet on YARN + +## How to create docker images to run MXNet on YARN + +Dockerfile to run MXNet on YARN needs two parts: + +**Base libraries which MXNet depends on** + +1) OS base image, for example ```ubuntu:18.04``` + +2) MXNet dependent libraries and packages. \ + For example ```python```, ```scipy```. For GPU support, you also need ```cuda```, ```cudnn```, etc. + +3) MXNet package. + +**Libraries to access HDFS** + +1) JDK + +2) Hadoop + +Here's an example of a base image (without GPU support) to install MXNet: +```shell +FROM ubuntu:18.04 + +# Install some development tools and packages +# MXNet 1.6 is going to be the last MXNet release to support Python2 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata git \ + wget zip python3 python3-pip python3-distutils libgomp1 libopenblas-dev libopencv-dev + +# Install latest MXNet using pip (without GPU support) +RUN pip3 install mxnet + +RUN echo "Install python related packages" && \ + pip3 install --user graphviz==0.8.4 ipykernel jupyter matplotlib numpy pandas scipy sklearn && \ + python3 -m ipykernel.kernelspec +``` + +On top of above image, add files, install packages to access HDFS +```shell +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN apt-get update && apt-get install -y openjdk-8-jdk wget + +# Install hadoop +ENV HADOOP_VERSION="3.1.2" +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +# If you are in mainland China, you can use the following command. +# RUN wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz +``` + +Build and push to your own docker registry: Use ```docker build ... ``` and ```docker push ...``` to finish this step. + +## Use examples to build your own MXNet docker images + +We provided some example Dockerfiles for you to build your own MXNet docker images. + +For latest MXNet + +- *docker/mxnet/base/ubuntu-18.04/Dockerfile.cpu.mxnet_latest*: Latest MXNet that supports CPU +- *docker/mxnet/base/ubuntu-18.04/Dockerfile.gpu.mxnet_latest*: Latest MXNet that supports GPU, which is prebuilt to CUDA10. + +# Build Docker images + +### Manually build Docker image: + +Under `docker/mxnet` directory, run `build-all.sh` to build all Docker images. This command will build the following Docker images: + +- `mxnet-latest-cpu-base:0.0.1` for base Docker image which includes Hadoop, MXNet +- `mxnet-latest-gpu-base:0.0.1` for base Docker image which includes Hadoop, MXNet, GPU base libraries. diff --git a/docs/userdocs/yarn/WriteDockerfilePT.md b/docs/userdocs/yarn/WriteDockerfilePT.md new file mode 100644 index 0000000000000000000000000000000000000000..837ac725f375e8946a8477c07b00328d5de659c9 --- /dev/null +++ b/docs/userdocs/yarn/WriteDockerfilePT.md @@ -0,0 +1,112 @@ + + +# Creating Docker Images for Running PyTorch on YARN + +## How to create docker images to run PyTorch on YARN + +Dockerfile to run PyTorch on YARN needs two parts: + +**Base libraries which PyTorch depends on** + +1) OS base image, for example ```ubuntu:18.04``` + +2) PyTorch dependent libraries and packages. For example ```python```, ```scipy```. For GPU support, you also need ```cuda```, ```cudnn```, etc. + +3) PyTorch package. + +**Libraries to access HDFS** + +1) JDK + +2) Hadoop + +Here's an example of a base image (with GPU support) to install PyTorch: +```shell +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 +ARG PYTHON_VERSION=3.6 +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + curl \ + vim \ + ca-certificates \ + libjpeg-dev \ + libpng-dev \ + wget &&\ + rm -rf /var/lib/apt/lists/* + + +RUN curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include cython typing && \ + /opt/conda/bin/conda install -y -c pytorch magma-cuda100 && \ + /opt/conda/bin/conda clean -ya +ENV PATH /opt/conda/bin:$PATH +RUN pip install ninja +# This must be done before pip so that requirements.txt is available +WORKDIR /opt/pytorch +RUN git clone https://github.com/pytorch/pytorch.git +WORKDIR pytorch +RUN git submodule update --init +RUN TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \ + CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \ + pip install -v . + +WORKDIR /opt/pytorch +RUN git clone https://github.com/pytorch/vision.git && cd vision && pip install -v . + +``` + +On top of above image, add files, install packages to access HDFS +```shell +RUN apt-get update && apt-get install -y openjdk-8-jdk wget +# Install hadoop +ENV HADOOP_VERSION="2.9.2" +RUN wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz +``` + +Build and push to your own docker registry: Use ```docker build ... ``` and ```docker push ...``` to finish this step. + +## Use examples to build your own PyTorch docker images + +We provided some example Dockerfiles for you to build your own PyTorch docker images. + +For latest PyTorch + +- *docker/pytorch/base/ubuntu-18.04/Dockerfile.gpu.pytorch_latest*: Latest Pytorch that supports GPU, which is prebuilt to CUDA10. +- *docker/pytorch/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.pytorch_latest*: Latest Pytorch that GPU, which is prebuilt to CUDA10, with models. + +## Build Docker images + +### Manually build Docker image: + +Under `docker/pytorch` directory, run `build-all.sh` to build all Docker images. This command will build the following Docker images: + +- `pytorch-latest-gpu-base:0.0.1` for base Docker image which includes Hadoop, PyTorch, GPU base libraries. +- `pytorch-latest-gpu:0.0.1` which includes cifar10 model as well + +### Use prebuilt images + +(No liability) +You can also use prebuilt images for convenience: + +- hadoopsubmarine/pytorch-latest-gpu-base:0.0.1 diff --git a/docs/userdocs/yarn/WriteDockerfileTF.md b/docs/userdocs/yarn/WriteDockerfileTF.md new file mode 100644 index 0000000000000000000000000000000000000000..e450f58506cd360c3d48a5b28b69fb356a3c13c9 --- /dev/null +++ b/docs/userdocs/yarn/WriteDockerfileTF.md @@ -0,0 +1,121 @@ + + +## Creating Docker Images for Running Tensorflow on YARN + +### How to create docker images to run Tensorflow on YARN + +Dockerfile to run Tensorflow on YARN need two part: + +**Base libraries which Tensorflow depends on** + +1) OS base image, for example ```ubuntu:18.04``` + +2) Tensorflow depended libraries and packages. For example ```python```, ```scipy```. For GPU support, need ```cuda```, ```cudnn```, etc. + +3) Tensorflow package. + +**Libraries to access HDFS** + +1) JDK + +2) Hadoop + +Here's an example of a base image (w/o GPU support) to install Tensorflow: +```shell +FROM ubuntu:18.04 + +# Pick up some TF dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + libfreetype6-dev \ + libpng-dev \ + libzmq3-dev \ + pkg-config \ + python \ + python-dev \ + rsync \ + software-properties-common \ + unzip \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -yq krb5-user libpam-krb5 && apt-get clean + +RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py && \ + rm get-pip.py + +RUN pip --no-cache-dir install \ + Pillow \ + h5py \ + ipykernel \ + jupyter \ + matplotlib \ + numpy \ + pandas \ + scipy \ + sklearn \ + && \ + python -m ipykernel.kernelspec + +RUN pip --no-cache-dir install \ + http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.13.1-cp27-none-linux_x86_64.whl +``` + +On top of above image, add files, install packages to access HDFS +```shell +RUN apt-get update && apt-get install -y openjdk-8-jdk wget +# Install hadoop +ENV HADOOP_VERSION="2.9.2" +RUN wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz +``` + +Build and push to your own docker registry: Use ```docker build ... ``` and ```docker push ...``` to finish this step. + +### Use examples to build your own Tensorflow docker images + +We provided following examples for you to build tensorflow docker images. + +For Tensorflow 1.13.1 (Precompiled to CUDA 10.x) + +- *docker/tensorflow/base/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1*: Tensorflow 1.13.1 supports CPU only. +- *docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1*: Tensorflow 1.13.1 supports CPU only, and included models +- *docker/tensorflow/base/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1*: Tensorflow 1.13.1 supports GPU, which is prebuilt to CUDA10. +- *docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1*: Tensorflow 1.13.1 supports GPU, which is prebuilt to CUDA10, with models. + +### Build Docker images + +#### Manually build Docker image: + +Under `docker/` directory, run `build-all.sh` to build Docker images. It will build following images: + +- `tf-1.13.1-gpu-base:0.0.1` for base Docker image which includes Hadoop, Tensorflow, GPU base libraries. +- `tf-1.13.1-gpu-base:0.0.1` for base Docker image which includes Hadoop. Tensorflow. +- `tf-1.13.1-gpu:0.0.1` which includes cifar10 model +- `tf-1.13.1-cpu:0.0.1` which inclues cifar10 model (cpu only). + +#### Use prebuilt images + +(No liability) +You can also use prebuilt images for convenience: + +- hadoopsubmarine/tf-1.13.1-gpu:0.0.1 +- hadoopsubmarine/tf-1.13.1-cpu:0.0.1 diff --git a/docs/userdocs/yarn/YARNRuntimeGuide.md b/docs/userdocs/yarn/YARNRuntimeGuide.md new file mode 100644 index 0000000000000000000000000000000000000000..ac93095751abc20c5f6bdec60b68acf4725600b3 --- /dev/null +++ b/docs/userdocs/yarn/YARNRuntimeGuide.md @@ -0,0 +1,310 @@ + + +# YARN Runtime Quick Start Guide + +## Prerequisite + +Check out the [Readme](README.md) + +## Build your own Docker image + +When you follow the documents below, and want to build your own Docker image for Tensorflow/PyTorch/MXNet? Please check out [Build your Docker image](Dockerfiles.md) for more details. + +## Launch TensorFlow Application: + +### Without Docker + +You need: + +* Build a Python virtual environment with TensorFlow 1.13.1 installed +* A cluster with Hadoop 2.9 or above. + +### Building a Python virtual environment with TensorFlow + +TonY requires a Python virtual environment zip with TensorFlow and any needed Python libraries already installed. + +``` +wget https://files.pythonhosted.org/packages/33/bc/fa0b5347139cd9564f0d44ebd2b147ac97c36b2403943dbee8a25fd74012/virtualenv-16.0.0.tar.gz +tar xf virtualenv-16.0.0.tar.gz + +# Make sure to install using Python 3, as TensorFlow only provides Python 3 artifacts +python virtualenv-16.0.0/virtualenv.py venv +. venv/bin/activate +pip install tensorflow==1.13.1 +zip -r myvenv.zip venv +deactivate +``` + +The above commands will produced a myvenv.zip and it will be used in below example. There's no need to copy it to other nodes. And it is not needed when using Docker to run the job. + + +**Note:** If you require a version of TensorFlow and TensorBoard prior to `1.13.1`, take a look at [this](https://github.com/linkedin/TonY/issues/42) issue. + + +### Get the training examples + +Get mnist_distributed.py from https://github.com/linkedin/TonY/tree/master/tony-examples/mnist-tensorflow + + +``` +SUBMARINE_VERSION= +SUBMARINE_HADOOP_VERSION=3.1 +CLASSPATH=$(hadoop classpath --glob):path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar \ +java org.apache.submarine.client.cli.Cli job run --name tf-job-001 \ + --framework tensorflow \ + --verbose \ + --input_path "" \ + --num_workers 2 \ + --worker_resources memory=1G,vcores=1 \ + --num_ps 1 \ + --ps_resources memory=1G,vcores=1 \ + --worker_launch_cmd "myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" \ + --ps_launch_cmd "myvenv.zip/venv/bin/python mnist_distributed.py --steps 2 --data_dir /tmp/data --working_dir /tmp/mode" \ + --insecure \ + --conf tony.containers.resources=path-to/myvenv.zip#archive,path-to/mnist_distributed.py,path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar +``` +You should then be able to see links and status of the jobs from command line: + +``` +2019-04-22 20:30:42,611 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000003/pi status: RUNNING +2019-04-22 20:30:42,612 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 1 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000004/pi status: RUNNING +2019-04-22 20:30:42,612 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: ps index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000002/pi status: RUNNING +2019-04-22 20:30:42,612 INFO tony.TonyClient: Logs for ps 0 at: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000002/pi +2019-04-22 20:30:42,612 INFO tony.TonyClient: Logs for worker 0 at: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000003/pi +2019-04-22 20:30:42,612 INFO tony.TonyClient: Logs for worker 1 at: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000004/pi +2019-04-22 20:30:44,625 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: ps index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000002/pi status: FINISHED +2019-04-22 20:30:44,625 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000003/pi status: FINISHED +2019-04-22 20:30:44,626 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 1 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000004/pi status: FINISHED + +``` + +### With Docker + +``` +SUBMARINE_VERSION= +SUBMARINE_HADOOP_VERSION=3.1 +CLASSPATH=$(hadoop classpath --glob):path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar \ +java org.apache.submarine.client.cli.Cli job run --name tf-job-001 \ + --framework tensorflow \ + --docker_image hadoopsubmarine/tf-1.8.0-cpu:0.0.1 \ + --input_path hdfs://pi-aw:9000/dataset/cifar-10-data \ + --worker_resources memory=3G,vcores=2 \ + --worker_launch_cmd "export CLASSPATH=\$(/hadoop-3.1.0/bin/hadoop classpath --glob) && cd /test/models/tutorials/image/cifar10_estimator && python cifar10_main.py --data-dir=%input_path% --job-dir=%checkpoint_path% --train-steps=10000 --eval-batch-size=16 --train-batch-size=16 --variable-strategy=CPU --num-gpus=0 --sync" \ + --env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ + --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 \ + --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ + --env HADOOP_HOME=/hadoop-3.1.0 \ + --env HADOOP_YARN_HOME=/hadoop-3.1.0 \ + --env HADOOP_COMMON_HOME=/hadoop-3.1.0 \ + --env HADOOP_HDFS_HOME=/hadoop-3.1.0 \ + --env HADOOP_CONF_DIR=/hadoop-3.1.0/etc/hadoop \ + --conf tony.containers.resources=path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar +``` +#### Notes: +1) `DOCKER_JAVA_HOME` points to JAVA_HOME inside Docker image. + +2) `DOCKER_HADOOP_HDFS_HOME` points to HADOOP_HDFS_HOME inside Docker image. + +We removed TonY submodule after applying [SUBMARINE-371](https://issues.apache.org/jira/browse/SUBMARINE-371) and changed to use TonY dependency directly. + +After Submarine v0.2.0, there is a uber jar `submarine-all-${SUBMARINE_VERSION}-hadoop-${HADOOP_VERSION}.jar` released together with +the `submarine-core-${SUBMARINE_VERSION}.jar`, `submarine-yarnservice-runtime-${SUBMARINE_VERSION}.jar` and `submarine-tony-runtime-${SUBMARINE_VERSION}.jar`. +
+ +## Launch PyTorch Application: + +### Without Docker + +You need: + +* Build a Python virtual environment with PyTorch 0.4.0+ installed +* A cluster with Hadoop 2.9 or above. + +### Building a Python virtual environment with PyTorch + +TonY requires a Python virtual environment zip with PyTorch and any needed Python libraries already installed. + +``` +wget https://files.pythonhosted.org/packages/33/bc/fa0b5347139cd9564f0d44ebd2b147ac97c36b2403943dbee8a25fd74012/virtualenv-16.0.0.tar.gz +tar xf virtualenv-16.0.0.tar.gz + +python virtualenv-16.0.0/virtualenv.py venv +. venv/bin/activate +pip install pytorch==0.4.0 +zip -r myvenv.zip venv +deactivate +``` + + +### Get the training examples + +Get mnist_distributed.py from https://github.com/linkedin/TonY/tree/master/tony-examples/mnist-pytorch + + +``` +SUBMARINE_VERSION= +SUBMARINE_HADOOP_VERSION=3.1 +CLASSPATH=$(hadoop classpath --glob):path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar \ +java org.apache.submarine.client.cli.Cli job run --name PyTorch-job-001 \ + --framework pytorch + --num_workers 2 \ + --worker_resources memory=3G,vcores=2 \ + --num_ps 2 \ + --ps_resources memory=3G,vcores=2 \ + --worker_launch_cmd "myvenv.zip/venv/bin/python mnist_distributed.py" \ + --ps_launch_cmd "myvenv.zip/venv/bin/python mnist_distributed.py" \ + --insecure \ + --conf tony.containers.resources=path-to/myvenv.zip#archive,path-to/mnist_distributed.py, \ +path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar +``` +You should then be able to see links and status of the jobs from command line: + +``` +2019-04-22 20:30:42,611 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000003/pi status: RUNNING +2019-04-22 20:30:42,612 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 1 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000004/pi status: RUNNING +2019-04-22 20:30:42,612 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: ps index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000002/pi status: RUNNING +2019-04-22 20:30:42,612 INFO tony.TonyClient: Logs for ps 0 at: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000002/pi +2019-04-22 20:30:42,612 INFO tony.TonyClient: Logs for worker 0 at: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000003/pi +2019-04-22 20:30:42,612 INFO tony.TonyClient: Logs for worker 1 at: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000004/pi +2019-04-22 20:30:44,625 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: ps index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000002/pi status: FINISHED +2019-04-22 20:30:44,625 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 0 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000003/pi status: FINISHED +2019-04-22 20:30:44,626 INFO tony.TonyClient: Tasks Status Updated: [TaskInfo] name: worker index: 1 url: http://pi-aw:8042/node/containerlogs/container_1555916523933_0030_01_000004/pi status: FINISHED + +``` + +### With Docker + +``` +SUBMARINE_VERSION= +SUBMARINE_HADOOP_VERSION=3.1 +CLASSPATH=$(hadoop classpath --glob):path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar \ +java org.apache.submarine.client.cli.Cli job run --name PyTorch-job-001 \ + --framework pytorch + --docker_image pytorch-latest-gpu:0.0.1 \ + --input_path "" \ + --num_workers 1 \ + --worker_resources memory=3G,vcores=2 \ + --worker_launch_cmd "cd /test/ && python cifar10_tutorial.py" \ + --env JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ + --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.2 \ + --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ + --env HADOOP_HOME=/hadoop-3.1.2 \ + --env HADOOP_YARN_HOME=/hadoop-3.1.2 \ + --env HADOOP_COMMON_HOME=/hadoop-3.1.2 \ + --env HADOOP_HDFS_HOME=/hadoop-3.1.2 \ + --env HADOOP_CONF_DIR=/hadoop-3.1.2/etc/hadoop \ + --conf tony.containers.resources=path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar +``` + +## Launch MXNet Application: + +### Without Docker + +You need: + +* Build a Python virtual environment with MXNet installed +* A cluster with Hadoop 2.9 or above. + +### Building a Python virtual environment with MXNet + +TonY requires a Python virtual environment zip with MXNet and any needed Python libraries already installed. + +``` +wget https://files.pythonhosted.org/packages/33/bc/fa0b5347139cd9564f0d44ebd2b147ac97c36b2403943dbee8a25fd74012/virtualenv-16.0.0.tar.gz +tar xf virtualenv-16.0.0.tar.gz + +python virtualenv-16.0.0/virtualenv.py venv +. venv/bin/activate +pip install mxnet==1.5.1 +zip -r myvenv.zip venv +deactivate +``` + + +### Get the training examples + +Get image_classification.py from this [link](https://github.com/apache/submarine/blob/master/dev-support/mini-submarine/submarine/image_classification.py) + + +``` +SUBMARINE_VERSION= +SUBMARINE_HADOOP_VERSION=3.1 +CLASSPATH=$(hadoop classpath --glob):path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar \ +java org.apache.submarine.client.cli.Cli job run --name MXNet-job-001 \ + --framework mxnet + --input_path "" \ + --num_workers 2 \ + --worker_resources memory=3G,vcores=2 \ + --worker_launch_cmd "myvenv.zip/venv/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --num_ps 2 \ + --ps_resources memory=3G,vcores=2 \ + --ps_launch_cmd "myvenv.zip/venv/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --num_schedulers=1 \ + --scheduler_resources memory=1G,vcores=1 \ + --scheduler_launch_cmd="myvenv.zip/venv/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --insecure \ + --conf tony.containers.resources=path-to/myvenv.zip#archive,path-to/image_classification.py, \ +path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar +``` +You should then be able to see links and status of the jobs from command line: + +``` +2020-04-16 20:23:43,834 INFO tony.TonyClient: Task status updated: [TaskInfo] name: server, index: 1, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000004/pi status: RUNNING +2020-04-16 20:23:43,834 INFO tony.TonyClient: Task status updated: [TaskInfo] name: server, index: 0, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000003/pi status: RUNNING +2020-04-16 20:23:43,834 INFO tony.TonyClient: Task status updated: [TaskInfo] name: worker, index: 1, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000006/pi status: RUNNING +2020-04-16 20:23:43,834 INFO tony.TonyClient: Task status updated: [TaskInfo] name: worker, index: 0, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000005/pi status: RUNNING +2020-04-16 20:23:43,834 INFO tony.TonyClient: Task status updated: [TaskInfo] name: scheduler, index: 0, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000002/pi status: RUNNING +2020-04-16 20:23:43,839 INFO tony.TonyClient: Logs for scheduler 0 at: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000002/pi +2020-04-16 20:23:43,839 INFO tony.TonyClient: Logs for server 0 at: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000003/pi +2020-04-16 20:23:43,840 INFO tony.TonyClient: Logs for server 1 at: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000004/pi +2020-04-16 20:23:43,840 INFO tony.TonyClient: Logs for worker 0 at: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000005/pi +2020-04-16 20:23:43,840 INFO tony.TonyClient: Logs for worker 1 at: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000006/pi +2020-04-16 21:02:09,723 INFO tony.TonyClient: Task status updated: [TaskInfo] name: scheduler, index: 0, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000002/pi status: SUCCEEDED +2020-04-16 21:02:09,736 INFO tony.TonyClient: Task status updated: [TaskInfo] name: worker, index: 0, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000005/pi status: SUCCEEDED +2020-04-16 21:02:09,737 INFO tony.TonyClient: Task status updated: [TaskInfo] name: server, index: 1, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000004/pi status: SUCCEEDED +2020-04-16 21:02:09,737 INFO tony.TonyClient: Task status updated: [TaskInfo] name: worker, index: 1, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000006/pi status: SUCCEEDED +2020-04-16 21:02:09,737 INFO tony.TonyClient: Task status updated: [TaskInfo] name: server, index: 0, url: http://pi-aw:8042/node/containerlogs/container_1587037749540_0005_01_000003/pi status: SUCCEEDED +``` + +### With Docker +You could refer to this [sample Dockerfile](project/github/submarine/docs/userdocs/yarn/docker/mxnet/cifar10/Dockerfile.cifar10.mx_1.5.1) for building your own Docker image. +``` +SUBMARINE_VERSION= +SUBMARINE_HADOOP_VERSION=3.1 +CLASSPATH=$(hadoop classpath --glob):path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar \ +java org.apache.submarine.client.cli.Cli job run --name MXNet-job-001 \ + --framework mxnet + --docker_image \ + --input_path "" \ + --num_schedulers 1 \ + --scheduler_resources memory=1G,vcores=1 \ + --scheduler_launch_cmd "/usr/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --num_workers 2 \ + --worker_resources memory=2G,vcores=1 \ + --worker_launch_cmd "/usr/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --num_ps 2 \ + --ps_resources memory=2G,vcores=1 \ + --ps_launch_cmd "/usr/bin/python image_classification.py --dataset cifar10 --model vgg11 --epochs 1 --kvstore dist_sync" \ + --verbose \ + --insecure \ + --conf tony.containers.resources=path-to/image_classification.py,path-to/submarine-all-${SUBMARINE_VERSION}-hadoop-${SUBMARINE_HADOOP_VERSION}.jar +``` + +## Use YARN Service to run Submarine: Deprecated + +Historically, Submarine supports to use [YARN Service](https://hadoop.apache.org/docs/r3.1.0/hadoop-yarn/hadoop-yarn-site/yarn-service/Overview.html) to submit deep learning jobs. Now we stop supporting it because YARN service is not actively developed by community, and extra dependencies such as RegistryDNS/ATS-v2 causes lots of issues for setup. + +As of now, you can still use YARN service to run Submarine, but code will be removed in the future release. We will only support use TonY when use Submarine on YARN. diff --git a/docs/userdocs/yarn/docker/mxnet/base/ubuntu-18.04/Dockerfile.cpu.mx_latest b/docs/userdocs/yarn/docker/mxnet/base/ubuntu-18.04/Dockerfile.cpu.mx_latest new file mode 100644 index 0000000000000000000000000000000000000000..d1f062dc320cd2f61142ae88c59e19e401b12b7f --- /dev/null +++ b/docs/userdocs/yarn/docker/mxnet/base/ubuntu-18.04/Dockerfile.cpu.mx_latest @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 + +# Install some development tools and packages +# MXNet 1.6 is going to be the last MXNet release to support Python2 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata git \ + wget zip python3 python3-pip python3-distutils libgomp1 libopenblas-dev libopencv-dev + +# Install latest MXNet using pip +RUN pip3 install mxnet + +# Install JDK +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN echo "$LOG_TAG Install java8" && \ + apt-get update && \ + apt-get install -y --no-install-recommends openjdk-8-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install Hadoop +WORKDIR / +RUN echo "Install Hadoop 3.1.2" +ENV HADOOP_VERSION="3.1.2" +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz + +RUN echo "Install python related packages" && \ + pip3 install --user graphviz==0.8.4 ipykernel jupyter matplotlib numpy pandas scipy sklearn && \ + python3 -m ipykernel.kernelspec + +# Set the locale to fix bash warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) +RUN apt-get update && apt-get install -y --no-install-recommends locales && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 diff --git a/docs/userdocs/yarn/docker/mxnet/base/ubuntu-18.04/Dockerfile.gpu.mx_latest b/docs/userdocs/yarn/docker/mxnet/base/ubuntu-18.04/Dockerfile.gpu.mx_latest new file mode 100644 index 0000000000000000000000000000000000000000..ecfc7527b7536cfe7de990ab7dc4a6467e901dd7 --- /dev/null +++ b/docs/userdocs/yarn/docker/mxnet/base/ubuntu-18.04/Dockerfile.gpu.mx_latest @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 + +# Install some development tools and packages +# MXNet 1.6 is going to be the last MXNet release to support Python2 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata git \ + wget zip python3 python3-pip python3-distutils libgomp1 libopenblas-dev libopencv-dev + +# Install latest MXNet with CUDA-10.0 using pip +RUN pip3 install mxnet-cu100 + +# Install JDK +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN echo "$LOG_TAG Install java8" && \ + apt-get update && \ + apt-get install -y --no-install-recommends openjdk-8-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install Hadoop +WORKDIR / +RUN echo "Install Hadoop 3.1.2" +ENV HADOOP_VERSION="3.1.2" +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz + +RUN echo "Install python related packages" && \ + pip3 install --user graphviz==0.8.4 ipykernel jupyter matplotlib numpy pandas scipy sklearn && \ + python3 -m ipykernel.kernelspec + +# Set the locale to fix bash warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) +RUN apt-get update && apt-get install -y --no-install-recommends locales && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 diff --git a/docs/userdocs/yarn/docker/mxnet/build-all.sh b/docs/userdocs/yarn/docker/mxnet/build-all.sh new file mode 100755 index 0000000000000000000000000000000000000000..5aab92249498b11bacbfd58fbe94ec53add2a645 --- /dev/null +++ b/docs/userdocs/yarn/docker/mxnet/build-all.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Building base images" + +set -euo pipefail + +cd base/ubuntu-18.04 + +docker build . -f Dockerfile.cpu.mx_latest -t mxnet-latest-cpu-base:0.0.1 +docker build . -f Dockerfile.gpu.mx_latest -t mxnet-latest-gpu-base:0.0.1 +echo "Finished building base images" diff --git a/docs/userdocs/yarn/docker/mxnet/cifar10/Dockerfile.cifar10.mx_1.5.1 b/docs/userdocs/yarn/docker/mxnet/cifar10/Dockerfile.cifar10.mx_1.5.1 new file mode 100644 index 0000000000000000000000000000000000000000..313b0f8d5daa0daeaaed2ecaf86d04fa1160600c --- /dev/null +++ b/docs/userdocs/yarn/docker/mxnet/cifar10/Dockerfile.cifar10.mx_1.5.1 @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 + +RUN apt-get update && apt-get install -y git wget zip python3 python3-pip \ + python3-distutils openjdk-8-jdk libgomp1 apt-transport-https ca-certificates curl \ + gnupg-agent software-properties-common + +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +RUN add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Install MXNet +RUN pip3 install "mxnet==1.5.1" + +# Install hadoop 3.1.0+ supported YARN service +ENV HADOOP_VERSION="3.1.2" +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +# If you are in mainland China, you can use the following command. +# RUN wget http://mirrors.shu.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + +RUN tar -xvf hadoop-${HADOOP_VERSION}.tar.gz -C /opt/ +RUN rm hadoop-${HADOOP_VERSION}.tar.gz + +# Copy the $HADOOP_CONF_DIR folder as "hadoop" folder in the same dir as dockerfile . +# ├── Dockerfile.cifar10_mx_1.5.1 +# └── hadoop +# ├── capacity-scheduler.xml +# ├── configuration.xsl +# ... +COPY hadoop /opt/hadoop-$HADOOP_VERSION/etc/hadoop + +# Config Hadoop env +ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION/ +ENV HADOOP_YARN_HOME=/opt/hadoop-$HADOOP_VERSION/ +ENV HADOOP_HDFS_HOME=/opt/hadoop-$HADOOP_VERSION/ +ENV HADOOP_CONF_DIR=/opt/hadoop-$HADOOP_VERSION/etc/hadoop +ENV HADOOP_COMMON_HOME=/opt/hadoop-$HADOOP_VERSION +ENV HADOOP_MAPRED_HOME=/opt/hadoop-$HADOOP_VERSION + +# Create a user, make sure the user groups are the same as your host +# and the container user's UID is same as your host's. +RUN groupadd -g 5000 hadoop +RUN useradd -u 1000 -g hadoop pi diff --git a/docs/userdocs/yarn/docker/pytorch/base/ubuntu-18.04/Dockerfile.gpu.pytorch_latest b/docs/userdocs/yarn/docker/pytorch/base/ubuntu-18.04/Dockerfile.gpu.pytorch_latest new file mode 100644 index 0000000000000000000000000000000000000000..8404b4269bc1af3ab48f2e101e86014d25f969c0 --- /dev/null +++ b/docs/userdocs/yarn/docker/pytorch/base/ubuntu-18.04/Dockerfile.gpu.pytorch_latest @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 +ARG PYTHON_VERSION=3.6 +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + curl \ + vim \ + ca-certificates \ + libjpeg-dev \ + libpng-dev \ + wget &&\ + rm -rf /var/lib/apt/lists/* + + +RUN curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include cython typing && \ + /opt/conda/bin/conda install -y -c pytorch magma-cuda100 && \ + /opt/conda/bin/conda clean -ya +ENV PATH /opt/conda/bin:$PATH +RUN pip install ninja +# This must be done before pip so that requirements.txt is available +WORKDIR /opt/pytorch +RUN git clone https://github.com/pytorch/pytorch.git +WORKDIR pytorch +RUN git submodule update --init +RUN TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \ + CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \ + pip install -v . + +WORKDIR /opt/pytorch +RUN git clone https://github.com/pytorch/vision.git && cd vision && pip install -v . + +WORKDIR / +# Install Hadoop +ENV HADOOP_VERSION="3.1.2" +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN echo "$LOG_TAG Install java8" && \ + apt-get update && \ + apt-get install -y --no-install-recommends openjdk-8-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN echo "Install python related packages" && \ + pip --no-cache-dir install Pillow h5py ipykernel jupyter matplotlib numpy pandas scipy sklearn && \ + python -m ipykernel.kernelspec + +# Set the locale to fix bash warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) +RUN apt-get update && apt-get install -y --no-install-recommends locales && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 + + +WORKDIR /workspace +RUN chmod -R a+w /workspace \ No newline at end of file diff --git a/docs/userdocs/yarn/docker/pytorch/build-all.sh b/docs/userdocs/yarn/docker/pytorch/build-all.sh new file mode 100755 index 0000000000000000000000000000000000000000..f0f795c02deb5098aa36ce793e835b3bc28f52d7 --- /dev/null +++ b/docs/userdocs/yarn/docker/pytorch/build-all.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Building base images" + +set -euo pipefail + +cd base/ubuntu-18.04 + +docker build . -f Dockerfile.gpu.pytorch_latest -t pytorch-latest-gpu-base:0.0.1 + +echo "Finished building base images" + +cd ../../with-cifar10-models/ubuntu-18.04 + +docker build . -f Dockerfile.gpu.pytorch_latest -t pytorch-latest-gpu:0.0.1 diff --git a/docs/userdocs/yarn/docker/pytorch/with-cifar10-models/cifar10_tutorial.py b/docs/userdocs/yarn/docker/pytorch/with-cifar10-models/cifar10_tutorial.py new file mode 100644 index 0000000000000000000000000000000000000000..02824eca56919013a814a89b13cf3a5a4f050f67 --- /dev/null +++ b/docs/userdocs/yarn/docker/pytorch/with-cifar10-models/cifar10_tutorial.py @@ -0,0 +1,354 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# -*- coding: utf-8 -*- +""" +Training a Classifier +===================== + +This is it. You have seen how to define neural networks, compute loss and make +updates to the weights of the network. + +Now you might be thinking, + +What about data? +---------------- + +Generally, when you have to deal with image, text, audio or video data, +you can use standard python packages that load data into a numpy array. +Then you can convert this array into a ``torch.*Tensor``. + +- For images, packages such as Pillow, OpenCV are useful +- For audio, packages such as scipy and librosa +- For text, either raw Python or Cython based loading, or NLTK and + SpaCy are useful + +Specifically for vision, we have created a package called +``torchvision``, that has data loaders for common datasets such as +Imagenet, CIFAR10, MNIST, etc. and data transformers for images, viz., +``torchvision.datasets`` and ``torch.utils.data.DataLoader``. + +This provides a huge convenience and avoids writing boilerplate code. + +For this tutorial, we will use the CIFAR10 dataset. +It has the classes: ‘airplane’, ‘automobile’, ‘bird’, ‘cat’, ‘deer’, +‘dog’, ‘frog’, ‘horse’, ‘ship’, ‘truck’. The images in CIFAR-10 are of +size 3x32x32, i.e. 3-channel color images of 32x32 pixels in size. + +.. figure:: /_static/img/cifar10.png + :alt: cifar10 + + cifar10 + + +Training an image classifier +---------------------------- + +We will do the following steps in order: + +1. Load and normalizing the CIFAR10 training and test datasets using + ``torchvision`` +2. Define a Convolutional Neural Network +3. Define a loss function +4. Train the network on the training data +5. Test the network on the test data + +1. Loading and normalizing CIFAR10 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using ``torchvision``, it’s extremely easy to load CIFAR10. +""" +import torch +import torchvision +import torchvision.transforms as transforms + +######################################################################## +# The output of torchvision datasets are PILImage images of range [0, 1]. +# We transform them to Tensors of normalized range [-1, 1]. + +transform = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + +trainset = torchvision.datasets.CIFAR10(root='./data', train=True, + download=True, transform=transform) +trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, + shuffle=True, num_workers=2) + +testset = torchvision.datasets.CIFAR10(root='./data', train=False, + download=True, transform=transform) +testloader = torch.utils.data.DataLoader(testset, batch_size=4, + shuffle=False, num_workers=2) + +classes = ('plane', 'car', 'bird', 'cat', + 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + +######################################################################## +# Let us show some of the training images, for fun. + +import matplotlib.pyplot as plt +import numpy as np + + +# functions to show an image + + +def imshow(img): + img = img / 2 + 0.5 # unnormalize + npimg = img.numpy() + plt.imshow(np.transpose(npimg, (1, 2, 0))) + plt.show() + + +# get some random training images +dataiter = iter(trainloader) +images, labels = dataiter.next() + +# show images +imshow(torchvision.utils.make_grid(images)) +# print labels +print(' '.join('%5s' % classes[labels[j]] for j in range(4))) + +######################################################################## +# 2. Define a Convolutional Neural Network +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Copy the neural network from the Neural Networks section before and modify it to +# take 3-channel images (instead of 1-channel images as it was defined). + +import torch.nn as nn +import torch.nn.functional as F + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +net = Net() + +######################################################################## +# 3. Define a Loss function and optimizer +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Let's use a Classification Cross-Entropy loss and SGD with momentum. + +import torch.optim as optim + +criterion = nn.CrossEntropyLoss() +optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) + +######################################################################## +# 4. Train the network +# ^^^^^^^^^^^^^^^^^^^^ +# +# This is when things start to get interesting. +# We simply have to loop over our data iterator, and feed the inputs to the +# network and optimize. + +for epoch in range(2): # loop over the dataset multiple times + + running_loss = 0.0 + for i, data in enumerate(trainloader, 0): + # get the inputs + inputs, labels = data + + # zero the parameter gradients + optimizer.zero_grad() + + # forward + backward + optimize + outputs = net(inputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # print statistics + running_loss += loss.item() + if i % 2000 == 1999: # print every 2000 mini-batches + print('[%d, %5d] loss: %.3f' % + (epoch + 1, i + 1, running_loss / 2000)) + running_loss = 0.0 + +print('Finished Training') + +######################################################################## +# 5. Test the network on the test data +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# We have trained the network for 2 passes over the training dataset. +# But we need to check if the network has learnt anything at all. +# +# We will check this by predicting the class label that the neural network +# outputs, and checking it against the ground-truth. If the prediction is +# correct, we add the sample to the list of correct predictions. +# +# Okay, first step. Let us display an image from the test set to get familiar. + +dataiter = iter(testloader) +images, labels = dataiter.next() + +# print images +imshow(torchvision.utils.make_grid(images)) +print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4))) + +######################################################################## +# Okay, now let us see what the neural network thinks these examples above are: + +outputs = net(images) + +######################################################################## +# The outputs are energies for the 10 classes. +# The higher the energy for a class, the more the network +# thinks that the image is of the particular class. +# So, let's get the index of the highest energy: +_, predicted = torch.max(outputs, 1) + +print('Predicted: ', ' '.join('%5s' % classes[predicted[j]] + for j in range(4))) + +######################################################################## +# The results seem pretty good. +# +# Let us look at how the network performs on the whole dataset. + +correct = 0 +total = 0 +with torch.no_grad(): + for data in testloader: + images, labels = data + outputs = net(images) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + +print('Accuracy of the network on the 10000 test images: %d %%' % ( + 100 * correct / total)) + +######################################################################## +# That looks waaay better than chance, which is 10% accuracy (randomly picking +# a class out of 10 classes). +# Seems like the network learnt something. +# +# Hmmm, what are the classes that performed well, and the classes that did +# not perform well: + +class_correct = list(0. for i in range(10)) +class_total = list(0. for i in range(10)) +with torch.no_grad(): + for data in testloader: + images, labels = data + outputs = net(images) + _, predicted = torch.max(outputs, 1) + c = (predicted == labels).squeeze() + for i in range(4): + label = labels[i] + class_correct[label] += c[i].item() + class_total[label] += 1 + +for i in range(10): + print('Accuracy of %5s : %2d %%' % ( + classes[i], 100 * class_correct[i] / class_total[i])) + +######################################################################## +# Okay, so what next? +# +# How do we run these neural networks on the GPU? +# +# Training on GPU +# ---------------- +# Just like how you transfer a Tensor onto the GPU, you transfer the neural +# net onto the GPU. +# +# Let's first define our device as the first visible cuda device if we have +# CUDA available: + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +# Assuming that we are on a CUDA machine, this should print a CUDA device: + +print(device) + +######################################################################## +# The rest of this section assumes that ``device`` is a CUDA device. +# +# Then these methods will recursively go over all modules and convert their +# parameters and buffers to CUDA tensors: +# +# .. code:: python +# +# net.to(device) +# +# +# Remember that you will have to send the inputs and targets at every step +# to the GPU too: +# +# .. code:: python +# +# inputs, labels = inputs.to(device), labels.to(device) +# +# Why dont I notice MASSIVE speedup compared to CPU? Because your network +# is realllly small. +# +# **Exercise:** Try increasing the width of your network (argument 2 of +# the first ``nn.Conv2d``, and argument 1 of the second ``nn.Conv2d`` – +# they need to be the same number), see what kind of speedup you get. +# +# **Goals achieved**: +# +# - Understanding PyTorch's Tensor library and neural networks at a high level. +# - Train a small neural network to classify images +# +# Training on multiple GPUs +# ------------------------- +# If you want to see even more MASSIVE speedup using all of your GPUs, +# please check out :doc:`data_parallel_tutorial`. +# +# Where do I go next? +# ------------------- +# +# - :doc:`Train neural nets to play video games ` +# - `Train a state-of-the-art ResNet network on imagenet`_ +# - `Train a face generator using Generative Adversarial Networks`_ +# - `Train a word-level language model using Recurrent LSTM networks`_ +# - `More examples`_ +# - `More tutorials`_ +# - `Discuss PyTorch on the Forums`_ +# - `Chat with other users on Slack`_ +# +# .. _Train a state-of-the-art ResNet network on imagenet: https://github.com/pytorch/examples/tree/master/imagenet +# .. _Train a face generator using Generative Adversarial Networks: https://github.com/pytorch/examples/tree/master/dcgan +# .. _Train a word-level language model using Recurrent LSTM networks: https://github.com/pytorch/examples/tree/master/word_language_model +# .. _More examples: https://github.com/pytorch/examples +# .. _More tutorials: https://github.com/pytorch/tutorials +# .. _Discuss PyTorch on the Forums: https://discuss.pytorch.org/ +# .. _Chat with other users on Slack: https://pytorch.slack.com/messages/beginner/ + +# %%%%%%INVISIBLE_CODE_BLOCK%%%%%% +del dataiter +# %%%%%%INVISIBLE_CODE_BLOCK%%%%%% diff --git a/docs/userdocs/yarn/docker/pytorch/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.pytorch_latest b/docs/userdocs/yarn/docker/pytorch/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.pytorch_latest new file mode 100644 index 0000000000000000000000000000000000000000..83e8fdecdd90c93d23d7be63fdaa30221f0bd8b5 --- /dev/null +++ b/docs/userdocs/yarn/docker/pytorch/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.pytorch_latest @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM pytorch-latest-gpu-base:0.0.1 + +RUN mkdir -p /test/data +RUN chmod -R 777 /test +ADD cifar10_tutorial.py /test/cifar10_tutorial.py \ No newline at end of file diff --git a/docs/userdocs/yarn/docker/tensorflow/base/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1 b/docs/userdocs/yarn/docker/tensorflow/base/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1 new file mode 100644 index 0000000000000000000000000000000000000000..0e575e685871eda3819109a808b51f25b2a6c48b --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/base/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1 @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 + +# Pick up some TF dependencies +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install -y --allow-downgrades --no-install-recommends \ + --allow-change-held-packages --allow-unauthenticated \ + build-essential libfreetype6-dev libpng-dev \ + libzmq3-dev pkg-config python python-dev \ + rsync software-properties-common curl unzip wget grep sed vim iputils-ping net-tools gdb python2.7-dbg tzdata && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -yq --no-install-recommends \ + krb5-user libpam-krb5 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py && \ + rm get-pip.py + +RUN echo "Install python related packages" && \ + apt-get update && \ + apt-get install -y --no-install-recommends gfortran \ + # numerical/algebra packages + libblas-dev libatlas-base-dev liblapack-dev \ + # font, image for matplotlib + libpng-dev libxft-dev \ + # for tkinter + python-tk libxml2-dev libxslt-dev zlib1g-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN pip --no-cache-dir install Pillow h5py ipykernel jupyter matplotlib numpy pandas scipy sklearn && \ + python -m ipykernel.kernelspec + +# Install TensorFlow CPU version. +ENV TENSORFLOW_VERSION="1.13.1" +RUN pip --no-cache-dir install \ + http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-${TENSORFLOW_VERSION}-cp27-none-linux_x86_64.whl +RUN apt-get update && apt-get install -y --no-install-recommends git && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install hadoop +ENV HADOOP_VERSION="3.1.2" +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +# If you are in mainland China, you can use the following command. +# RUN wget http://mirrors.shu.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN echo "$LOG_TAG Install java8" && \ + apt-get update && \ + apt-get install -y --no-install-recommends openjdk-8-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Set the locale to fix bash warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) +RUN apt-get update && apt-get install -y --no-install-recommends locales && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 diff --git a/docs/userdocs/yarn/docker/tensorflow/base/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1 b/docs/userdocs/yarn/docker/tensorflow/base/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1 new file mode 100644 index 0000000000000000000000000000000000000000..dc7027c0fb256e8c317f8db0a34d5eeb7b51106b --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/base/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1 @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 + +# Pick up some TF dependencies +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install -y --allow-downgrades --no-install-recommends \ + --allow-change-held-packages --allow-unauthenticated \ + build-essential libfreetype6-dev \ + libzmq3-dev pkg-config python python-dev \ + rsync software-properties-common curl unzip wget grep sed vim \ + iputils-ping net-tools gdb python2.7-dbg tzdata \ + cuda-command-line-tools-10-0 cuda-cublas-10-0 \ + cuda-cufft-10-0 cuda-curand-10-0 cuda-cusolver-10-0 \ + cuda-cusparse-10-0 libcudnn7=7.4.1.5-1+cuda10.0 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install TensorRT +RUN apt-get update && \ + apt-get install -y --allow-unauthenticated --no-install-recommends \ + nvinfer-runtime-trt-repo-ubuntu1804-5.0.2-ga-cuda10.0 && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + libnvinfer5=5.0.2-1+cuda10.0 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + + +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install -yq --no-install-recommends krb5-user libpam-krb5 \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py && \ + rm get-pip.py + +RUN echo "Install python related packages" && \ + apt-get -y update && \ + apt-get install -y --no-install-recommends gfortran \ + # numerical/algebra packages + libblas-dev libatlas-base-dev liblapack-dev \ + # font, image for matplotlib + libpng-dev libxft-dev \ + # for tkinter + python-tk libxml2-dev libxslt-dev zlib1g-dev && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN pip --no-cache-dir install Pillow h5py ipykernel jupyter matplotlib numpy pandas scipy sklearn && \ + python -m ipykernel.kernelspec + +# Install TensorFlow GPU version. +ENV TENSORFLOW_VERSION="1.13.1" +RUN pip --no-cache-dir install \ + http://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${TENSORFLOW_VERSION}-cp27-none-linux_x86_64.whl +RUN apt-get update && apt-get install -y --no-install-recommends git && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install hadoop +ENV HADOOP_VERSION="3.1.2" +RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz +# If you are in mainland China, you can use the following command. +# RUN wget http://mirrors.shu.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + +RUN tar zxf hadoop-${HADOOP_VERSION}.tar.gz +RUN ln -s hadoop-${HADOOP_VERSION} hadoop-current +RUN rm hadoop-${HADOOP_VERSION}.tar.gz + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN echo "$LOG_TAG Install java8" && \ + apt-get -y update && \ + apt-get install -y --no-install-recommends openjdk-8-jdk && \ + rm -rf /var/lib/apt/lists/* + +# Set the locale to fix bash warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) +RUN apt-get update && apt-get install -y --no-install-recommends locales && \ + apt-get clean && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 diff --git a/docs/userdocs/yarn/docker/tensorflow/build-all.sh b/docs/userdocs/yarn/docker/tensorflow/build-all.sh new file mode 100755 index 0000000000000000000000000000000000000000..5c02379fb913d9ee5df852d545d8b88cb1b17718 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/build-all.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Building base images" + +set -euo pipefail + +cd base/ubuntu-18.04 + +docker build . -f Dockerfile.cpu.tf_1.13.1 -t tf-1.13.1-cpu-base:0.0.1 +docker build . -f Dockerfile.gpu.tf_1.13.1 -t tf-1.13.1-gpu-base:0.0.1 + +echo "Finished building base images" + +cd ../../with-cifar10-models/ubuntu-18.04 + +docker build . -f Dockerfile.cpu.tf_1.13.1 -t tf-1.13.1-cpu:0.0.1 +docker build . -f Dockerfile.gpu.tf_1.13.1 -t tf-1.13.1-gpu:0.0.1 + +cd ../../mnist +docker build . -f Dockerfile.tony.tf.mnist.tf_1.13.1 -t tony-mnist-tf-1.13.1:0.0.1 diff --git a/docs/userdocs/yarn/docker/tensorflow/mnist/Dockerfile.tony.tf.mnist.tf_1.13.1 b/docs/userdocs/yarn/docker/tensorflow/mnist/Dockerfile.tony.tf.mnist.tf_1.13.1 new file mode 100644 index 0000000000000000000000000000000000000000..86874d47e714000d6361f57e8a2aee05d83f4d45 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/mnist/Dockerfile.tony.tf.mnist.tf_1.13.1 @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 +ENV HADOOP_VERSION 2.9.2 +ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz +RUN apt-get update +RUN apt-get -y install apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + git \ + software-properties-common \ + openjdk-8-jdk vim \ + wget python3-distutils +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +RUN add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +# Download Hadoop binaries. +RUN set -x \ + && curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \ + && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \ + && rm /tmp/hadoop.tar.gz* + +# Copy the $HADOOP_CONF_DIR folder as "hadoop" folder in the same dir as dockerfile +# pi@pi-aw:~/apache/submarine/docker$ tree +# . +# ├── Dockerfile +# └── hadoopconf +# ├── capacity-scheduler.xml +# ├── configuration.xsl +# ... +COPY hadoopconf /opt/hadoop-$HADOOP_VERSION/etc/hadoop + +# Download and config submarine +RUN cd ~ +RUN git clone https://github.com/apache/submarine.git +RUN cd submarine + +RUN submarine/dev-support/mini-submarine/submarine/build_python_virtual_env.sh +RUN mv venv/ /opt/ +RUN chmod +r -R /opt/venv + +# Config Hadoop env +ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION/ +ENV HADOOP_YARN_HOME=/opt/hadoop-$HADOOP_VERSION/ +ENV HADOOP_HDFS_HOME=/opt/hadoop-$HADOOP_VERSION/ +ENV HADOOP_CONF_DIR=/opt/hadoop-$HADOOP_VERSION/etc/hadoop +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + +# Crate user, make sure the user groups are the same as your host +RUN groupadd -g 5000 hadoop +RUN useradd -u 1000 -g hadoop pi +RUN mkdir /home/pi +RUN chown pi:hadoop /home/pi +RUN mkdir /tmp/mode +RUN chmod 777 /tmp/mode diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1 b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1 new file mode 100644 index 0000000000000000000000000000000000000000..188e4878b9b945a64dc83f2dde298205a5ab92a4 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.cpu.tf_1.13.1 @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM tf-1.13.1-cpu-base:0.0.1 + +# Include models +RUN mkdir /test +ADD cifar10_estimator_tf_1.13.1 /test/cifar10_estimator +RUN chown -R nobody /test \ No newline at end of file diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1 b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1 new file mode 100644 index 0000000000000000000000000000000000000000..8819fa619f095119b9e967bfbdbab88295a71828 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/Dockerfile.gpu.tf_1.13.1 @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM tf-1.13.1-gpu-base:0.0.1 + +# Include models +RUN mkdir /test +ADD cifar10_estimator_tf_1.13.1 /test/cifar10_estimator +RUN chown -R nobody /test \ No newline at end of file diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/README.md b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4ca03fa2ca147c05e75211f4812a642a30bee06e --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/README.md @@ -0,0 +1,542 @@ + + +(Copied from https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator) + +CIFAR-10 is a common benchmark in machine learning for image recognition. + +http://www.cs.toronto.edu/~kriz/cifar.html + +Code in this directory focuses on how to use TensorFlow Estimators to train and +evaluate a CIFAR-10 ResNet model on: + +* A single host with one CPU; +* A single host with multiple GPUs; +* Multiple hosts with CPU or multiple GPUs; + +Before trying to run the model we highly encourage you to read all the README. + +## Prerequisite + +1. [Install](https://www.tensorflow.org/install/) TensorFlow version 1.2.1 or +later. + +2. Download the CIFAR-10 dataset and generate TFRecord files using the provided +script. The script and associated command below will download the CIFAR-10 +dataset and then generate a TFRecord for the training, validation, and +evaluation datasets. + +```shell +python generate_cifar10_tfrecords.py --data-dir=${PWD}/cifar-10-data +``` + +After running the command above, you should see the following files in the +--data-dir (```ls -R cifar-10-data```): + +* train.tfrecords +* validation.tfrecords +* eval.tfrecords + + +## Training on a single machine with GPUs or CPU + +Run the training on CPU only. After training, it runs the evaluation. + +``` +python cifar10_main.py --data-dir=${PWD}/cifar-10-data \ + --job-dir=/tmp/cifar10 \ + --num-gpus=0 \ + --train-steps=1000 +``` + +Run the model on 2 GPUs using CPU as parameter server. After training, it runs +the evaluation. +``` +python cifar10_main.py --data-dir=${PWD}/cifar-10-data \ + --job-dir=/tmp/cifar10 \ + --num-gpus=2 \ + --train-steps=1000 +``` + +Run the model on 2 GPUs using GPU as parameter server. +It will run an experiment, which for local setting basically means it will run +stop training +a couple of times to perform evaluation. + +``` +python cifar10_main.py --data-dir=${PWD}/cifar-10-data \ + --job-dir=/tmp/cifar10 \ + --variable-strategy GPU \ + --num-gpus=2 \ +``` + +There are more command line flags to play with; run +`python cifar10_main.py --help` for details. + +## Run distributed training + +### (Optional) Running on Google Cloud Machine Learning Engine + +This example can be run on Google Cloud Machine Learning Engine (ML Engine), +which will configure the environment and take care of running workers, +parameters servers, and masters in a fault tolerant way. + +To install the command line tool, and set up a project and billing, see the +quickstart [here](https://cloud.google.com/ml-engine/docs/quickstarts/command-line). + +You'll also need a Google Cloud Storage bucket for the data. If you followed the +instructions above, you can just run: + +``` +MY_BUCKET=gs:// +gsutil cp -r ${PWD}/cifar-10-data $MY_BUCKET/ +``` + +Then run the following command from the `tutorials/image` directory of this +repository (the parent directory of this README): + +``` +gcloud ml-engine jobs submit training cifarmultigpu \ + --runtime-version 1.2 \ + --job-dir=$MY_BUCKET/model_dirs/cifarmultigpu \ + --config cifar10_estimator/cmle_config.yaml \ + --package-path cifar10_estimator/ \ + --module-name cifar10_estimator.cifar10_main \ + -- \ + --data-dir=$MY_BUCKET/cifar-10-data \ + --num-gpus=4 \ + --train-steps=1000 +``` + + +### Set TF_CONFIG + +Considering that you already have multiple hosts configured, all you need is a +`TF_CONFIG` environment variable on each host. You can set up the hosts manually +or check [tensorflow/ecosystem](https://github.com/tensorflow/ecosystem) for +instructions about how to set up a Cluster. + +The `TF_CONFIG` will be used by the `RunConfig` to know the existing hosts and +their task: `master`, `ps` or `worker`. + +Here's an example of `TF_CONFIG`. + +```python +cluster = {'master': ['master-ip:8000'], + 'ps': ['ps-ip:8000'], + 'worker': ['worker-ip:8000']} + +TF_CONFIG = json.dumps( + {'cluster': cluster, + 'task': {'type': master, 'index': 0}, + 'model_dir': 'gs:///', + 'environment': 'cloud' + }) +``` + +*Cluster* + +A cluster spec, which is basically a dictionary that describes all of the tasks +in the cluster. More about it [here](https://www.tensorflow.org/deploy/distributed). + +In this cluster spec we are defining a cluster with 1 master, 1 ps and 1 worker. + +* `ps`: saves the parameters among all workers. All workers can + read/write/update the parameters for model via ps. As some models are + extremely large the parameters are shared among the ps (each ps stores a + subset). + +* `worker`: does the training. + +* `master`: basically a special worker, it does training, but also restores and + saves checkpoints and do evaluation. + +*Task* + +The Task defines what is the role of the current node, for this example the node +is the master on index 0 on the cluster spec, the task will be different for +each node. An example of the `TF_CONFIG` for a worker would be: + +```python +cluster = {'master': ['master-ip:8000'], + 'ps': ['ps-ip:8000'], + 'worker': ['worker-ip:8000']} + +TF_CONFIG = json.dumps( + {'cluster': cluster, + 'task': {'type': worker, 'index': 0}, + 'model_dir': 'gs:///', + 'environment': 'cloud' + }) +``` + +*Model_dir* + +This is the path where the master will save the checkpoints, graph and +TensorBoard files. For a multi host environment you may want to use a +Distributed File System, Google Storage and DFS are supported. + +*Environment* + +By the default environment is *local*, for a distributed setting we need to +change it to *cloud*. + +### Running script + +Once you have a `TF_CONFIG` configured properly on each host you're ready to run +on distributed settings. + +#### Master +Run this on master: +Runs an Experiment in sync mode on 4 GPUs using CPU as parameter server for +40000 steps. It will run evaluation a couple of times during training. The +num_workers argument is used only to update the learning rate correctly. Make +sure the model_dir is the same as defined on the TF_CONFIG. + +```shell +python cifar10_main.py --data-dir=gs://path/cifar-10-data \ + --job-dir=gs://path/model_dir/ \ + --num-gpus=4 \ + --train-steps=40000 \ + --sync \ + --num-workers=2 +``` + +*Output:* + +```shell +INFO:tensorflow:Using model_dir in TF_CONFIG: gs://path/model_dir/ +INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_num_ps_replicas': 1, '_keep_checkpoint_max': 5, '_task_type': u'master', '_is_chief': True, '_cluster_spec': , '_model_dir': 'gs://path/model_dir/', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_session_config': intra_op_parallelism_threads: 1 +gpu_options { +} +allow_soft_placement: true +, '_tf_random_seed': None, '_environment': u'cloud', '_num_worker_replicas': 1, '_task_id': 0, '_save_summary_steps': 100, '_tf_config': gpu_options { + per_process_gpu_memory_fraction: 1.0 +} +, '_evaluation_master': '', '_master': u'grpc://master-ip:8000'} +... +2017-08-01 19:59:26.496208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: +name: Tesla K80 +major: 3 minor: 7 memoryClockRate (GHz) 0.8235 +pciBusID 0000:00:04.0 +Total memory: 11.17GiB +Free memory: 11.09GiB +2017-08-01 19:59:26.775660: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 1 with properties: +name: Tesla K80 +major: 3 minor: 7 memoryClockRate (GHz) 0.8235 +pciBusID 0000:00:05.0 +Total memory: 11.17GiB +Free memory: 11.10GiB +... +2017-08-01 19:59:29.675171: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000 +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: (?, 16, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: (?, 32, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: (?, 64) +INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: (?, 11) +INFO:tensorflow:SyncReplicasV2: replicas_to_aggregate=1; total_num_replicas=1 +INFO:tensorflow:Create CheckpointSaverHook. +INFO:tensorflow:Restoring parameters from gs://path/model_dir/model.ckpt-0 +2017-08-01 19:59:37.560775: I tensorflow/core/distributed_runtime/master_session.cc:999] Start master session 156fcb55fe6648d6 with config: +intra_op_parallelism_threads: 1 +gpu_options { + per_process_gpu_memory_fraction: 1 +} +allow_soft_placement: true + +INFO:tensorflow:Saving checkpoints for 1 into gs://path/model_dir/model.ckpt. +INFO:tensorflow:loss = 1.20682, step = 1 +INFO:tensorflow:loss = 1.20682, learning_rate = 0.1 +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: (?, 16, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: (?, 32, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: (?, 64) +INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: (?, 11) +INFO:tensorflow:SyncReplicasV2: replicas_to_aggregate=2; total_num_replicas=2 +INFO:tensorflow:Starting evaluation at 2017-08-01-20:00:14 +2017-08-01 20:00:15.745881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0) +2017-08-01 20:00:15.745949: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:1) -> (device: 1, name: Tesla K80, pci bus id: 0000:00:05.0) +2017-08-01 20:00:15.745958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:2) -> (device: 2, name: Tesla K80, pci bus id: 0000:00:06.0) +2017-08-01 20:00:15.745964: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:3) -> (device: 3, name: Tesla K80, pci bus id: 0000:00:07.0) +2017-08-01 20:00:15.745969: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:4) -> (device: 4, name: Tesla K80, pci bus id: 0000:00:08.0) +2017-08-01 20:00:15.745975: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:5) -> (device: 5, name: Tesla K80, pci bus id: 0000:00:09.0) +2017-08-01 20:00:15.745987: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:6) -> (device: 6, name: Tesla K80, pci bus id: 0000:00:0a.0) +2017-08-01 20:00:15.745997: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:7) -> (device: 7, name: Tesla K80, pci bus id: 0000:00:0b.0) +INFO:tensorflow:Restoring parameters from gs://path/model_dir/model.ckpt-10023 +INFO:tensorflow:Evaluation [1/100] +INFO:tensorflow:Evaluation [2/100] +INFO:tensorflow:Evaluation [3/100] +INFO:tensorflow:Evaluation [4/100] +INFO:tensorflow:Evaluation [5/100] +INFO:tensorflow:Evaluation [6/100] +INFO:tensorflow:Evaluation [7/100] +INFO:tensorflow:Evaluation [8/100] +INFO:tensorflow:Evaluation [9/100] +INFO:tensorflow:Evaluation [10/100] +INFO:tensorflow:Evaluation [11/100] +INFO:tensorflow:Evaluation [12/100] +INFO:tensorflow:Evaluation [13/100] +... +INFO:tensorflow:Evaluation [100/100] +INFO:tensorflow:Finished evaluation at 2017-08-01-20:00:31 +INFO:tensorflow:Saving dict for global step 1: accuracy = 0.0994, global_step = 1, loss = 630.425 +``` + +#### Worker + +Run this on worker: +Runs an Experiment in sync mode on 4 GPUs using CPU as parameter server for +40000 steps. It will run evaluation a couple of times during training. Make sure +the model_dir is the same as defined on the TF_CONFIG. + +```shell +python cifar10_main.py --data-dir=gs://path/cifar-10-data \ + --job-dir=gs://path/model_dir/ \ + --num-gpus=4 \ + --train-steps=40000 \ + --sync +``` + +*Output:* + +```shell +INFO:tensorflow:Using model_dir in TF_CONFIG: gs://path/model_dir/ +INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, +'_num_ps_replicas': 1, '_keep_checkpoint_max': 5, '_task_type': u'worker', +'_is_chief': False, '_cluster_spec': +, +'_model_dir': 'gs:///model_dir/', +'_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, +'_session_config': intra_op_parallelism_threads: 1 +gpu_options { +} +allow_soft_placement: true +, '_tf_random_seed': None, '_environment': u'cloud', '_num_worker_replicas': 1, +'_task_id': 0, '_save_summary_steps': 100, '_tf_config': gpu_options { + per_process_gpu_memory_fraction: 1.0 + } +... +2017-08-01 19:59:26.496208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties: +name: Tesla K80 +major: 3 minor: 7 memoryClockRate (GHz) 0.8235 +pciBusID 0000:00:04.0 +Total memory: 11.17GiB +Free memory: 11.09GiB +2017-08-01 19:59:26.775660: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 1 with properties: +name: Tesla K80 +major: 3 minor: 7 memoryClockRate (GHz) 0.8235 +pciBusID 0000:00:05.0 +Total memory: 11.17GiB +Free memory: 11.10GiB +... +2017-08-01 19:59:29.675171: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000 +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: (?, 16, 32, 32) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: (?, 16, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: (?, 32, 16, 16) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: (?, 32, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: (?, 64, 8, 8) +INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: (?, 64) +INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: (?, 11) +INFO:tensorflow:SyncReplicasV2: replicas_to_aggregate=2; total_num_replicas=2 +INFO:tensorflow:Create CheckpointSaverHook. +2017-07-31 22:38:04.629150: I +tensorflow/core/distributed_runtime/master.cc:209] CreateSession still waiting +for response from worker: /job:master/replica:0/task:0 +2017-07-31 22:38:09.263492: I +tensorflow/core/distributed_runtime/master_session.cc:999] Start master +session cc58f93b1e259b0c with config: +intra_op_parallelism_threads: 1 +gpu_options { +per_process_gpu_memory_fraction: 1 +} +allow_soft_placement: true +INFO:tensorflow:loss = 5.82382, step = 0 +INFO:tensorflow:loss = 5.82382, learning_rate = 0.8 +INFO:tensorflow:Average examples/sec: 1116.92 (1116.92), step = 10 +INFO:tensorflow:Average examples/sec: 1233.73 (1377.83), step = 20 +INFO:tensorflow:Average examples/sec: 1485.43 (2509.3), step = 30 +INFO:tensorflow:Average examples/sec: 1680.27 (2770.39), step = 40 +INFO:tensorflow:Average examples/sec: 1825.38 (2788.78), step = 50 +INFO:tensorflow:Average examples/sec: 1929.32 (2697.27), step = 60 +INFO:tensorflow:Average examples/sec: 2015.17 (2749.05), step = 70 +INFO:tensorflow:loss = 37.6272, step = 79 (19.554 sec) +INFO:tensorflow:loss = 37.6272, learning_rate = 0.8 (19.554 sec) +INFO:tensorflow:Average examples/sec: 2074.92 (2618.36), step = 80 +INFO:tensorflow:Average examples/sec: 2132.71 (2744.13), step = 90 +INFO:tensorflow:Average examples/sec: 2183.38 (2777.21), step = 100 +INFO:tensorflow:Average examples/sec: 2224.4 (2739.03), step = 110 +INFO:tensorflow:Average examples/sec: 2240.28 (2431.26), step = 120 +INFO:tensorflow:Average examples/sec: 2272.12 (2739.32), step = 130 +INFO:tensorflow:Average examples/sec: 2300.68 (2750.03), step = 140 +INFO:tensorflow:Average examples/sec: 2325.81 (2745.63), step = 150 +INFO:tensorflow:Average examples/sec: 2347.14 (2721.53), step = 160 +INFO:tensorflow:Average examples/sec: 2367.74 (2754.54), step = 170 +INFO:tensorflow:loss = 27.8453, step = 179 (18.893 sec) +... +``` + +#### PS + +Run this on ps: +The ps will not do training so most of the arguments won't affect the execution + +```shell +python cifar10_main.py --job-dir=gs://path/model_dir/ +``` + +*Output:* + +```shell +INFO:tensorflow:Using model_dir in TF_CONFIG: gs://path/model_dir/ +INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_num_ps_replicas': 1, '_keep_checkpoint_max': 5, '_task_type': u'ps', '_is_chief': False, '_cluster_spec': , '_model_dir': 'gs://path/model_dir/', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_session_config': intra_op_parallelism_threads: 1 +gpu_options { +} +allow_soft_placement: true +, '_tf_random_seed': None, '_environment': u'cloud', '_num_worker_replicas': 1, '_task_id': 0, '_save_summary_steps': 100, '_tf_config': gpu_options { + per_process_gpu_memory_fraction: 1.0 +} +, '_evaluation_master': '', '_master': u'grpc://master-ip:8000'} +2017-07-31 22:54:58.928088: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job master -> {0 -> master-ip:8000} +2017-07-31 22:54:58.928153: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:8000} +2017-07-31 22:54:58.928160: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> worker-ip:8000} +2017-07-31 22:54:58.929873: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000 +``` + +## Visualizing results with TensorBoard + +When using Estimators you can also visualize your data in TensorBoard, with no +changes in your code. You can use TensorBoard to visualize your TensorFlow +graph, plot quantitative metrics about the execution of your graph, and show +additional data like images that pass through it. + +You'll see something similar to this if you "point" TensorBoard to the +`job dir` parameter you used to train or evaluate your model. + +Check TensorBoard during training or after it. Just point TensorBoard to the +model_dir you chose on the previous step. + +```shell +tensorboard --log-dir="" +``` + +## Warnings + +When running `cifar10_main.py` with `--sync` argument you may see an error +similar to: + +```python +File "cifar10_main.py", line 538, in + tf.app.run() +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 48, in run + _sys.exit(main(_sys.argv[:1] + flags_passthrough)) +File "cifar10_main.py", line 518, in main + hooks), run_config=config) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 210, in run + return _execute_schedule(experiment, schedule) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 47, in _execute_schedule + return task() +File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 501, in train_and_evaluate + hooks=self._eval_hooks) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 681, in _call_evaluate + hooks=hooks) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py", line 292, in evaluate + name=name) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py", line 638, in _evaluate_model + features, labels, model_fn_lib.ModeKeys.EVAL) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py", line 545, in _call_model_fn + features=features, labels=labels, **kwargs) +File "cifar10_main.py", line 331, in _resnet_model_fn + gradvars, global_step=tf.train.get_global_step()) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/sync_replicas_optimizer.py", line 252, in apply_gradients + variables.global_variables()) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py", line 170, in wrapped + return _add_should_use_warning(fn(*args, **kwargs)) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py", line 139, in _add_should_use_warning + wrapped = TFShouldUseWarningWrapper(x) +File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py", line 96, in __init__ + stack = [s.strip() for s in traceback.format_stack()] +``` + +This should not affect your training, and should be fixed on the next releases. diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10.py b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10.py new file mode 100644 index 0000000000000000000000000000000000000000..5e1a70895ad8c89333325b6a4c1666bbf528ed37 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10.py @@ -0,0 +1,113 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""CIFAR-10 data set. + +See http://www.cs.toronto.edu/~kriz/cifar.html. +""" +import os + +import tensorflow as tf + +HEIGHT = 32 +WIDTH = 32 +DEPTH = 3 + + +class Cifar10DataSet(object): + """Cifar10 data set. + + Described by http://www.cs.toronto.edu/~kriz/cifar.html. + """ + + def __init__(self, data_dir, subset='train', use_distortion=True): + self.data_dir = data_dir + self.subset = subset + self.use_distortion = use_distortion + + def get_filenames(self): + if self.subset in ['train', 'validation', 'eval']: + return [os.path.join(self.data_dir, self.subset + '.tfrecords')] + else: + raise ValueError('Invalid data subset "%s"' % self.subset) + + def parser(self, serialized_example): + """Parses a single tf.Example into image and label tensors.""" + # Dimensions of the images in the CIFAR-10 dataset. + # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the + # input format. + features = tf.parse_single_example( + serialized_example, + features={ + 'image': tf.FixedLenFeature([], tf.string), + 'label': tf.FixedLenFeature([], tf.int64), + }) + image = tf.decode_raw(features['image'], tf.uint8) + image.set_shape([DEPTH * HEIGHT * WIDTH]) + + # Reshape from [depth * height * width] to [depth, height, width]. + image = tf.cast( + tf.transpose(tf.reshape(image, [DEPTH, HEIGHT, WIDTH]), [1, 2, 0]), + tf.float32) + label = tf.cast(features['label'], tf.int32) + + # Custom preprocessing. + image = self.preprocess(image) + + return image, label + + def make_batch(self, batch_size): + """Read the images and labels from 'filenames'.""" + filenames = self.get_filenames() + # Repeat infinitely. + dataset = tf.data.TFRecordDataset(filenames).repeat() + + # Parse records. + dataset = dataset.map( + self.parser, num_parallel_calls=batch_size) + + # Potentially shuffle records. + if self.subset == 'train': + min_queue_examples = int( + Cifar10DataSet.num_examples_per_epoch(self.subset) * 0.4) + # Ensure that the capacity is sufficiently large to provide good random + # shuffling. + dataset = dataset.shuffle(buffer_size=min_queue_examples + 3 * batch_size) + + # Batch it up. + dataset = dataset.batch(batch_size) + iterator = dataset.make_one_shot_iterator() + image_batch, label_batch = iterator.get_next() + + return image_batch, label_batch + + def preprocess(self, image): + """Preprocess a single image in [height, width, depth] layout.""" + if self.subset == 'train' and self.use_distortion: + # Pad 4 pixels on each dimension of feature map, done in mini-batch + image = tf.image.resize_image_with_crop_or_pad(image, 40, 40) + image = tf.random_crop(image, [HEIGHT, WIDTH, DEPTH]) + image = tf.image.random_flip_left_right(image) + return image + + @staticmethod + def num_examples_per_epoch(subset='train'): + if subset == 'train': + return 45000 + elif subset == 'validation': + return 5000 + elif subset == 'eval': + return 10000 + else: + raise ValueError('Invalid data subset "%s"' % subset) diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_main.py b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_main.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd2418905440ee089ea3885d3623c83ebd6de48 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_main.py @@ -0,0 +1,521 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ResNet model for classifying images from CIFAR-10 dataset. + +Support single-host training with one or multiple devices. + +ResNet as proposed in: +Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun +Deep Residual Learning for Image Recognition. arXiv:1512.03385 + +CIFAR-10 as in: +http://www.cs.toronto.edu/~kriz/cifar.html + + +""" +from __future__ import division +from __future__ import print_function + +import argparse +import functools +import itertools +import os + +import cifar10 +import cifar10_model +import cifar10_utils +import numpy as np +import six +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + +tf.logging.set_verbosity(tf.logging.INFO) + + +def get_model_fn(num_gpus, variable_strategy, num_workers): + """Returns a function that will build the resnet model.""" + + def _resnet_model_fn(features, labels, mode, params): + """Resnet model body. + + Support single host, one or more GPU training. Parameter distribution can + be either one of the following scheme. + 1. CPU is the parameter server and manages gradient updates. + 2. Parameters are distributed evenly across all GPUs, and the first GPU + manages gradient updates. + + Args: + features: a list of tensors, one for each tower + labels: a list of tensors, one for each tower + mode: ModeKeys.TRAIN or EVAL + params: Hyperparameters suitable for tuning + Returns: + A EstimatorSpec object. + """ + is_training = (mode == tf.estimator.ModeKeys.TRAIN) + weight_decay = params.weight_decay + momentum = params.momentum + + tower_features = features + tower_labels = labels + tower_losses = [] + tower_gradvars = [] + tower_preds = [] + + # channels first (NCHW) is normally optimal on GPU and channels last (NHWC) + # on CPU. The exception is Intel MKL on CPU which is optimal with + # channels_last. + data_format = params.data_format + if not data_format: + if num_gpus == 0: + data_format = 'channels_last' + else: + data_format = 'channels_first' + + if num_gpus == 0: + num_devices = 1 + device_type = 'cpu' + else: + num_devices = num_gpus + device_type = 'gpu' + + for i in range(num_devices): + worker_device = '/{}:{}'.format(device_type, i) + if variable_strategy == 'CPU': + device_setter = cifar10_utils.local_device_setter( + worker_device=worker_device) + elif variable_strategy == 'GPU': + device_setter = cifar10_utils.local_device_setter( + ps_device_type='gpu', + worker_device=worker_device, + ps_strategy=tf.contrib.training.GreedyLoadBalancingStrategy( + num_gpus, tf.contrib.training.byte_size_load_fn)) + with tf.variable_scope('resnet', reuse=bool(i != 0)): + with tf.name_scope('tower_%d' % i) as name_scope: + with tf.device(device_setter): + loss, gradvars, preds = _tower_fn( + is_training, weight_decay, tower_features[i], tower_labels[i], + data_format, params.num_layers, params.batch_norm_decay, + params.batch_norm_epsilon) + tower_losses.append(loss) + tower_gradvars.append(gradvars) + tower_preds.append(preds) + if i == 0: + # Only trigger batch_norm moving mean and variance update from + # the 1st tower. Ideally, we should grab the updates from all + # towers but these stats accumulate extremely fast so we can + # ignore the other stats from the other towers without + # significant detriment. + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, + name_scope) + + # Now compute global loss and gradients. + gradvars = [] + with tf.name_scope('gradient_averaging'): + all_grads = {} + for grad, var in itertools.chain(*tower_gradvars): + if grad is not None: + all_grads.setdefault(var, []).append(grad) + for var, grads in six.iteritems(all_grads): + # Average gradients on the same device as the variables + # to which they apply. + with tf.device(var.device): + if len(grads) == 1: + avg_grad = grads[0] + else: + avg_grad = tf.multiply(tf.add_n(grads), 1. / len(grads)) + gradvars.append((avg_grad, var)) + + # Device that runs the ops to apply global gradient updates. + consolidation_device = '/gpu:0' if variable_strategy == 'GPU' else '/cpu:0' + with tf.device(consolidation_device): + # Suggested learning rate scheduling from + # https://github.com/ppwwyyxx/tensorpack/blob/master/examples/ResNet/cifar10-resnet.py#L155 + num_batches_per_epoch = cifar10.Cifar10DataSet.num_examples_per_epoch( + 'train') // (params.train_batch_size * num_workers) + boundaries = [ + num_batches_per_epoch * x + for x in np.array([82, 123, 300], dtype=np.int64) + ] + staged_lr = [params.learning_rate * x for x in [1, 0.1, 0.01, 0.002]] + + learning_rate = tf.train.piecewise_constant(tf.train.get_global_step(), + boundaries, staged_lr) + + loss = tf.reduce_mean(tower_losses, name='loss') + + examples_sec_hook = cifar10_utils.ExamplesPerSecondHook( + params.train_batch_size, every_n_steps=10) + + tensors_to_log = {'learning_rate': learning_rate, 'loss': loss} + + logging_hook = tf.train.LoggingTensorHook( + tensors=tensors_to_log, every_n_iter=100) + + train_hooks = [logging_hook, examples_sec_hook] + + optimizer = tf.train.MomentumOptimizer( + learning_rate=learning_rate, momentum=momentum) + + if params.sync: + optimizer = tf.train.SyncReplicasOptimizer( + optimizer, replicas_to_aggregate=num_workers) + sync_replicas_hook = optimizer.make_session_run_hook(params.is_chief) + train_hooks.append(sync_replicas_hook) + + # Create single grouped train op + train_op = [ + optimizer.apply_gradients( + gradvars, global_step=tf.train.get_global_step()) + ] + train_op.extend(update_ops) + train_op = tf.group(*train_op) + + predictions = { + 'classes': + tf.concat([p['classes'] for p in tower_preds], axis=0), + 'probabilities': + tf.concat([p['probabilities'] for p in tower_preds], axis=0) + } + stacked_labels = tf.concat(labels, axis=0) + metrics = { + 'accuracy': + tf.metrics.accuracy(stacked_labels, predictions['classes']) + } + + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + loss=loss, + train_op=train_op, + training_hooks=train_hooks, + eval_metric_ops=metrics) + + return _resnet_model_fn + + +def _tower_fn(is_training, weight_decay, feature, label, data_format, + num_layers, batch_norm_decay, batch_norm_epsilon): + """Build computation tower (Resnet). + + Args: + is_training: true if is training graph. + weight_decay: weight regularization strength, a float. + feature: a Tensor. + label: a Tensor. + data_format: channels_last (NHWC) or channels_first (NCHW). + num_layers: number of layers, an int. + batch_norm_decay: decay for batch normalization, a float. + batch_norm_epsilon: epsilon for batch normalization, a float. + + Returns: + A tuple with the loss for the tower, the gradients and parameters, and + predictions. + + """ + model = cifar10_model.ResNetCifar10( + num_layers, + batch_norm_decay=batch_norm_decay, + batch_norm_epsilon=batch_norm_epsilon, + is_training=is_training, + data_format=data_format) + logits = model.forward_pass(feature, input_data_format='channels_last') + tower_pred = { + 'classes': tf.argmax(input=logits, axis=1), + 'probabilities': tf.nn.softmax(logits) + } + + tower_loss = tf.losses.sparse_softmax_cross_entropy( + logits=logits, labels=label) + tower_loss = tf.reduce_mean(tower_loss) + + model_params = tf.trainable_variables() + tower_loss += weight_decay * tf.add_n( + [tf.nn.l2_loss(v) for v in model_params]) + + tower_grad = tf.gradients(tower_loss, model_params) + + return tower_loss, zip(tower_grad, model_params), tower_pred + + +def input_fn(data_dir, + subset, + num_shards, + batch_size, + use_distortion_for_training=True): + """Create input graph for model. + + Args: + data_dir: Directory where TFRecords representing the dataset are located. + subset: one of 'train', 'validate' and 'eval'. + num_shards: num of towers participating in data-parallel training. + batch_size: total batch size for training to be divided by the number of + shards. + use_distortion_for_training: True to use distortions. + Returns: + two lists of tensors for features and labels, each of num_shards length. + """ + with tf.device('/cpu:0'): + use_distortion = subset == 'train' and use_distortion_for_training + dataset = cifar10.Cifar10DataSet(data_dir, subset, use_distortion) + image_batch, label_batch = dataset.make_batch(batch_size) + if num_shards <= 1: + # No GPU available or only 1 GPU. + return [image_batch], [label_batch] + + # Note that passing num=batch_size is safe here, even though + # dataset.batch(batch_size) can, in some cases, return fewer than batch_size + # examples. This is because it does so only when repeating for a limited + # number of epochs, but our dataset repeats forever. + image_batch = tf.unstack(image_batch, num=batch_size, axis=0) + label_batch = tf.unstack(label_batch, num=batch_size, axis=0) + feature_shards = [[] for i in range(num_shards)] + label_shards = [[] for i in range(num_shards)] + for i in xrange(batch_size): + idx = i % num_shards + feature_shards[idx].append(image_batch[i]) + label_shards[idx].append(label_batch[i]) + feature_shards = [tf.parallel_stack(x) for x in feature_shards] + label_shards = [tf.parallel_stack(x) for x in label_shards] + return feature_shards, label_shards + + +def get_experiment_fn(data_dir, + num_gpus, + variable_strategy, + use_distortion_for_training=True): + """Returns an Experiment function. + + Experiments perform training on several workers in parallel, + in other words experiments know how to invoke train and eval in a sensible + fashion for distributed training. Arguments passed directly to this + function are not tunable, all other arguments should be passed within + tf.HParams, passed to the enclosed function. + + Args: + data_dir: str. Location of the data for input_fns. + num_gpus: int. Number of GPUs on each worker. + variable_strategy: String. CPU to use CPU as the parameter server + and GPU to use the GPUs as the parameter server. + use_distortion_for_training: bool. See cifar10.Cifar10DataSet. + Returns: + A function (tf.estimator.RunConfig, tf.contrib.training.HParams) -> + tf.contrib.learn.Experiment. + + Suitable for use by tf.contrib.learn.learn_runner, which will run various + methods on Experiment (train, evaluate) based on information + about the current runner in `run_config`. + """ + + def _experiment_fn(run_config, hparams): + """Returns an Experiment.""" + # Create estimator. + train_input_fn = functools.partial( + input_fn, + data_dir, + subset='train', + num_shards=num_gpus, + batch_size=hparams.train_batch_size, + use_distortion_for_training=use_distortion_for_training) + + eval_input_fn = functools.partial( + input_fn, + data_dir, + subset='eval', + batch_size=hparams.eval_batch_size, + num_shards=num_gpus) + + num_eval_examples = cifar10.Cifar10DataSet.num_examples_per_epoch('eval') + if num_eval_examples % hparams.eval_batch_size != 0: + raise ValueError( + 'validation set size must be multiple of eval_batch_size') + + train_steps = hparams.train_steps + eval_steps = num_eval_examples // hparams.eval_batch_size + + classifier = tf.estimator.Estimator( + model_fn=get_model_fn(num_gpus, variable_strategy, + run_config.num_worker_replicas or 1), + config=run_config, + params=hparams) + + # Create experiment. + return tf.contrib.learn.Experiment( + classifier, + train_input_fn=train_input_fn, + eval_input_fn=eval_input_fn, + train_steps=train_steps, + eval_steps=eval_steps) + + return _experiment_fn + + +def main(job_dir, data_dir, num_gpus, variable_strategy, + use_distortion_for_training, log_device_placement, num_intra_threads, + **hparams): + # The env variable is on deprecation path, default is set to off. + os.environ['TF_SYNC_ON_FINISH'] = '0' + os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' + + # Session configuration. + sess_config = tf.ConfigProto( + allow_soft_placement=True, + log_device_placement=log_device_placement, + intra_op_parallelism_threads=num_intra_threads, + gpu_options=tf.GPUOptions(force_gpu_compatible=True)) + + config = cifar10_utils.RunConfig( + session_config=sess_config, model_dir=job_dir) + tf.contrib.learn.learn_runner.run( + get_experiment_fn(data_dir, num_gpus, variable_strategy, + use_distortion_for_training), + run_config=config, + hparams=tf.contrib.training.HParams( + is_chief=config.is_chief, + **hparams)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--data-dir', + type=str, + required=True, + help='The directory where the CIFAR-10 input data is stored.') + parser.add_argument( + '--job-dir', + type=str, + required=True, + help='The directory where the model will be stored.') + parser.add_argument( + '--variable-strategy', + choices=['CPU', 'GPU'], + type=str, + default='CPU', + help='Where to locate variable operations') + parser.add_argument( + '--num-gpus', + type=int, + default=1, + help='The number of gpus used. Uses only CPU if set to 0.') + parser.add_argument( + '--num-layers', + type=int, + default=44, + help='The number of layers of the model.') + parser.add_argument( + '--train-steps', + type=int, + default=80000, + help='The number of steps to use for training.') + parser.add_argument( + '--train-batch-size', + type=int, + default=128, + help='Batch size for training.') + parser.add_argument( + '--eval-batch-size', + type=int, + default=100, + help='Batch size for validation.') + parser.add_argument( + '--momentum', + type=float, + default=0.9, + help='Momentum for MomentumOptimizer.') + parser.add_argument( + '--weight-decay', + type=float, + default=2e-4, + help='Weight decay for convolutions.') + parser.add_argument( + '--learning-rate', + type=float, + default=0.1, + help="""\ + This is the initial learning rate value. The learning rate will decrease + during training. For more details check the model_fn implementation in + this file.\ + """) + parser.add_argument( + '--use-distortion-for-training', + type=bool, + default=True, + help='If doing image distortion for training.') + parser.add_argument( + '--sync', + action='store_true', + default=False, + help="""\ + If present when running in a distributed environment will run on sync mode.\ + """) + parser.add_argument( + '--num-intra-threads', + type=int, + default=0, + help="""\ + Number of threads to use for intra-op parallelism. When training on CPU + set to 0 to have the system pick the appropriate number or alternatively + set it to the number of physical CPU cores.\ + """) + parser.add_argument( + '--num-inter-threads', + type=int, + default=0, + help="""\ + Number of threads to use for inter-op parallelism. If set to 0, the + system will pick an appropriate number.\ + """) + parser.add_argument( + '--data-format', + type=str, + default=None, + help="""\ + If not set, the data format best for the training device is used. + Allowed values: channels_first (NCHW) channels_last (NHWC).\ + """) + parser.add_argument( + '--log-device-placement', + action='store_true', + default=False, + help='Whether to log device placement.') + parser.add_argument( + '--batch-norm-decay', + type=float, + default=0.997, + help='Decay for batch norm.') + parser.add_argument( + '--batch-norm-epsilon', + type=float, + default=1e-5, + help='Epsilon for batch norm.') + args = parser.parse_args() + + if args.num_gpus > 0: + assert tf.test.is_gpu_available(), "Requested GPUs but none found." + if args.num_gpus < 0: + raise ValueError( + 'Invalid GPU count: \"--num-gpus\" must be 0 or a positive integer.') + if args.num_gpus == 0 and args.variable_strategy == 'GPU': + raise ValueError('num-gpus=0, CPU must be used as parameter server. Set' + '--variable-strategy=CPU.') + if (args.num_layers - 2) % 6 != 0: + raise ValueError('Invalid --num-layers parameter.') + if args.num_gpus != 0 and args.train_batch_size % args.num_gpus != 0: + raise ValueError('--train-batch-size must be multiple of --num-gpus.') + if args.num_gpus != 0 and args.eval_batch_size % args.num_gpus != 0: + raise ValueError('--eval-batch-size must be multiple of --num-gpus.') + + main(**vars(args)) diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_model.py b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_model.py new file mode 100644 index 0000000000000000000000000000000000000000..d67c233dbba5558482156333eb229dc723fe4a7c --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_model.py @@ -0,0 +1,80 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model class for Cifar10 Dataset.""" +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +import model_base + + +class ResNetCifar10(model_base.ResNet): + """Cifar10 model with ResNetV1 and basic residual block.""" + + def __init__(self, + num_layers, + is_training, + batch_norm_decay, + batch_norm_epsilon, + data_format='channels_first'): + super(ResNetCifar10, self).__init__( + is_training, + data_format, + batch_norm_decay, + batch_norm_epsilon + ) + self.n = (num_layers - 2) // 6 + # Add one in case label starts with 1. No impact if label starts with 0. + self.num_classes = 10 + 1 + self.filters = [16, 16, 32, 64] + self.strides = [1, 2, 2] + + def forward_pass(self, x, input_data_format='channels_last'): + """Build the core model within the graph.""" + if self._data_format != input_data_format: + if input_data_format == 'channels_last': + # Computation requires channels_first. + x = tf.transpose(x, [0, 3, 1, 2]) + else: + # Computation requires channels_last. + x = tf.transpose(x, [0, 2, 3, 1]) + + # Image standardization. + x = x / 128 - 1 + + x = self._conv(x, 3, 16, 1) + x = self._batch_norm(x) + x = self._relu(x) + + # Use basic (non-bottleneck) block and ResNet V1 (post-activation). + res_func = self._residual_v1 + + # 3 stages of block stacking. + for i in range(3): + with tf.name_scope('stage'): + for j in range(self.n): + if j == 0: + # First block in a stage, filters and strides may change. + x = res_func(x, 3, self.filters[i], self.filters[i + 1], + self.strides[i]) + else: + # Following blocks in a stage, constant filters and unit stride. + x = res_func(x, 3, self.filters[i + 1], self.filters[i + 1], 1) + + x = self._global_avg_pool(x) + x = self._fully_connected(x, self.num_classes) + + return x diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_utils.py b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d3d29dd82ec7782908b9542bde5b3e0013b418c3 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/cifar10_utils.py @@ -0,0 +1,153 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import collections +import six + +import tensorflow as tf + +from tensorflow.python.platform import tf_logging as logging +from tensorflow.core.framework import node_def_pb2 +from tensorflow.python.framework import device as pydev +from tensorflow.python.training import basic_session_run_hooks +from tensorflow.python.training import session_run_hook +from tensorflow.python.training import training_util +from tensorflow.python.training import device_setter +from tensorflow.contrib.learn.python.learn import run_config + + +# TODO(b/64848083) Remove once uid bug is fixed +class RunConfig(tf.contrib.learn.RunConfig): + def uid(self, whitelist=None): + """Generates a 'Unique Identifier' based on all internal fields. + Caller should use the uid string to check `RunConfig` instance integrity + in one session use, but should not rely on the implementation details, which + is subject to change. + Args: + whitelist: A list of the string names of the properties uid should not + include. If `None`, defaults to `_DEFAULT_UID_WHITE_LIST`, which + includes most properties user allowed to change. + Returns: + A uid string. + """ + if whitelist is None: + whitelist = run_config._DEFAULT_UID_WHITE_LIST + + state = {k: v for k, v in self.__dict__.items() if not k.startswith('__')} + # Pop out the keys in whitelist. + for k in whitelist: + state.pop('_' + k, None) + + ordered_state = collections.OrderedDict( + sorted(state.items(), key=lambda t: t[0])) + # For class instance without __repr__, some special cares are required. + # Otherwise, the object address will be used. + if '_cluster_spec' in ordered_state: + ordered_state['_cluster_spec'] = collections.OrderedDict( + sorted(ordered_state['_cluster_spec'].as_dict().items(), + key=lambda t: t[0]) + ) + return ', '.join( + '%s=%r' % (k, v) for (k, v) in six.iteritems(ordered_state)) + + +class ExamplesPerSecondHook(session_run_hook.SessionRunHook): + """Hook to print out examples per second. + + Total time is tracked and then divided by the total number of steps + to get the average step time and then batch_size is used to determine + the running average of examples per second. The examples per second for the + most recent interval is also logged. + """ + + def __init__( + self, + batch_size, + every_n_steps=100, + every_n_secs=None,): + """Initializer for ExamplesPerSecondHook. + + Args: + batch_size: Total batch size used to calculate examples/second from + global time. + every_n_steps: Log stats every n steps. + every_n_secs: Log stats every n seconds. + """ + if (every_n_steps is None) == (every_n_secs is None): + raise ValueError('exactly one of every_n_steps' + ' and every_n_secs should be provided.') + self._timer = basic_session_run_hooks.SecondOrStepTimer( + every_steps=every_n_steps, every_secs=every_n_secs) + + self._step_train_time = 0 + self._total_steps = 0 + self._batch_size = batch_size + + def begin(self): + self._global_step_tensor = training_util.get_global_step() + if self._global_step_tensor is None: + raise RuntimeError( + 'Global step should be created to use StepCounterHook.') + + def before_run(self, run_context): # pylint: disable=unused-argument + return basic_session_run_hooks.SessionRunArgs(self._global_step_tensor) + + def after_run(self, run_context, run_values): + _ = run_context + + global_step = run_values.results + if self._timer.should_trigger_for_step(global_step): + elapsed_time, elapsed_steps = self._timer.update_last_triggered_step( + global_step) + if elapsed_time is not None: + steps_per_sec = elapsed_steps / elapsed_time + self._step_train_time += elapsed_time + self._total_steps += elapsed_steps + + average_examples_per_sec = self._batch_size * ( + self._total_steps / self._step_train_time) + current_examples_per_sec = steps_per_sec * self._batch_size + # Average examples/sec followed by current examples/sec + logging.info('%s: %g (%g), step = %g', 'Average examples/sec', + average_examples_per_sec, current_examples_per_sec, + self._total_steps) + +def local_device_setter(num_devices=1, + ps_device_type='cpu', + worker_device='/cpu:0', + ps_ops=None, + ps_strategy=None): + if ps_ops == None: + ps_ops = ['Variable', 'VariableV2', 'VarHandleOp'] + + if ps_strategy is None: + ps_strategy = device_setter._RoundRobinStrategy(num_devices) + if not six.callable(ps_strategy): + raise TypeError("ps_strategy must be callable") + + def _local_device_chooser(op): + current_device = pydev.DeviceSpec.from_string(op.device or "") + + node_def = op if isinstance(op, node_def_pb2.NodeDef) else op.node_def + if node_def.op in ps_ops: + ps_device_spec = pydev.DeviceSpec.from_string( + '/{}:{}'.format(ps_device_type, ps_strategy(op))) + + ps_device_spec.merge_from(current_device) + return ps_device_spec.to_string() + else: + worker_device_spec = pydev.DeviceSpec.from_string(worker_device or "") + worker_device_spec.merge_from(current_device) + return worker_device_spec.to_string() + return _local_device_chooser diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/generate_cifar10_tfrecords.py b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/generate_cifar10_tfrecords.py new file mode 100644 index 0000000000000000000000000000000000000000..d1a599c31bfbe5f5efea9d983d958d75dbcc26d1 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/generate_cifar10_tfrecords.py @@ -0,0 +1,118 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Read CIFAR-10 data from pickled numpy arrays and writes TFRecords. + +Generates tf.train.Example protos and writes them to TFRecord files from the +python version of the CIFAR-10 dataset downloaded from +https://www.cs.toronto.edu/~kriz/cifar.html. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import os +import sys + +import tarfile +from six.moves import cPickle as pickle +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + +CIFAR_FILENAME = 'cifar-10-python.tar.gz' +CIFAR_DOWNLOAD_URL = 'https://www.cs.toronto.edu/~kriz/' + CIFAR_FILENAME +CIFAR_LOCAL_FOLDER = 'cifar-10-batches-py' + + +def download_and_extract(data_dir): + # download CIFAR-10 if not already downloaded. + tf.contrib.learn.datasets.base.maybe_download(CIFAR_FILENAME, data_dir, + CIFAR_DOWNLOAD_URL) + tarfile.open(os.path.join(data_dir, CIFAR_FILENAME), + 'r:gz').extractall(data_dir) + + +def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + + +def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + +def _get_file_names(): + """Returns the file names expected to exist in the input_dir.""" + file_names = {} + file_names['train'] = ['data_batch_%d' % i for i in xrange(1, 5)] + file_names['validation'] = ['data_batch_5'] + file_names['eval'] = ['test_batch'] + return file_names + + +def read_pickle_from_file(filename): + with tf.gfile.Open(filename, 'rb') as f: + if sys.version_info >= (3, 0): + data_dict = pickle.load(f, encoding='bytes') + else: + data_dict = pickle.load(f) + return data_dict + + +def convert_to_tfrecord(input_files, output_file): + """Converts a file to TFRecords.""" + print('Generating %s' % output_file) + with tf.python_io.TFRecordWriter(output_file) as record_writer: + for input_file in input_files: + data_dict = read_pickle_from_file(input_file) + data = data_dict[b'data'] + labels = data_dict[b'labels'] + num_entries_in_batch = len(labels) + for i in range(num_entries_in_batch): + example = tf.train.Example(features=tf.train.Features( + feature={ + 'image': _bytes_feature(data[i].tobytes()), + 'label': _int64_feature(labels[i]) + })) + record_writer.write(example.SerializeToString()) + + +def main(data_dir): + print('Download from {} and extract.'.format(CIFAR_DOWNLOAD_URL)) + download_and_extract(data_dir) + file_names = _get_file_names() + input_dir = os.path.join(data_dir, CIFAR_LOCAL_FOLDER) + for mode, files in file_names.items(): + input_files = [os.path.join(input_dir, f) for f in files] + output_file = os.path.join(data_dir, mode + '.tfrecords') + try: + os.remove(output_file) + except OSError: + pass + # Convert to tf.train.Example and write the to TFRecords. + convert_to_tfrecord(input_files, output_file) + print('Done!') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--data-dir', + type=str, + default='', + help='Directory to download and extract CIFAR-10 to.') + + args = parser.parse_args() + main(args.data_dir) diff --git a/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/model_base.py b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/model_base.py new file mode 100644 index 0000000000000000000000000000000000000000..35e52b8355d05afa81f33fdde482fb61a0a19bc5 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/with-cifar10-models/ubuntu-18.04/cifar10_estimator_tf_1.13.1/model_base.py @@ -0,0 +1,219 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ResNet model. + +Related papers: +https://arxiv.org/pdf/1603.05027v2.pdf +https://arxiv.org/pdf/1512.03385v1.pdf +https://arxiv.org/pdf/1605.07146v1.pdf +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + + +class ResNet(object): + """ResNet model.""" + + def __init__(self, is_training, data_format, batch_norm_decay, batch_norm_epsilon): + """ResNet constructor. + + Args: + is_training: if build training or inference model. + data_format: the data_format used during computation. + one of 'channels_first' or 'channels_last'. + """ + self._batch_norm_decay = batch_norm_decay + self._batch_norm_epsilon = batch_norm_epsilon + self._is_training = is_training + assert data_format in ('channels_first', 'channels_last') + self._data_format = data_format + + def forward_pass(self, x): + raise NotImplementedError( + 'forward_pass() is implemented in ResNet sub classes') + + def _residual_v1(self, + x, + kernel_size, + in_filter, + out_filter, + stride, + activate_before_residual=False): + """Residual unit with 2 sub layers, using Plan A for shortcut connection.""" + + del activate_before_residual + with tf.name_scope('residual_v1') as name_scope: + orig_x = x + + x = self._conv(x, kernel_size, out_filter, stride) + x = self._batch_norm(x) + x = self._relu(x) + + x = self._conv(x, kernel_size, out_filter, 1) + x = self._batch_norm(x) + + if in_filter != out_filter: + orig_x = self._avg_pool(orig_x, stride, stride) + pad = (out_filter - in_filter) // 2 + if self._data_format == 'channels_first': + orig_x = tf.pad(orig_x, [[0, 0], [pad, pad], [0, 0], [0, 0]]) + else: + orig_x = tf.pad(orig_x, [[0, 0], [0, 0], [0, 0], [pad, pad]]) + + x = self._relu(tf.add(x, orig_x)) + + tf.logging.info('image after unit %s: %s', name_scope, x.get_shape()) + return x + + def _residual_v2(self, + x, + in_filter, + out_filter, + stride, + activate_before_residual=False): + """Residual unit with 2 sub layers with preactivation, plan A shortcut.""" + + with tf.name_scope('residual_v2') as name_scope: + if activate_before_residual: + x = self._batch_norm(x) + x = self._relu(x) + orig_x = x + else: + orig_x = x + x = self._batch_norm(x) + x = self._relu(x) + + x = self._conv(x, 3, out_filter, stride) + + x = self._batch_norm(x) + x = self._relu(x) + x = self._conv(x, 3, out_filter, [1, 1, 1, 1]) + + if in_filter != out_filter: + pad = (out_filter - in_filter) // 2 + orig_x = self._avg_pool(orig_x, stride, stride) + if self._data_format == 'channels_first': + orig_x = tf.pad(orig_x, [[0, 0], [pad, pad], [0, 0], [0, 0]]) + else: + orig_x = tf.pad(orig_x, [[0, 0], [0, 0], [0, 0], [pad, pad]]) + + x = tf.add(x, orig_x) + + tf.logging.info('image after unit %s: %s', name_scope, x.get_shape()) + return x + + def _bottleneck_residual_v2(self, + x, + in_filter, + out_filter, + stride, + activate_before_residual=False): + """Bottleneck residual unit with 3 sub layers, plan B shortcut.""" + + with tf.name_scope('bottle_residual_v2') as name_scope: + if activate_before_residual: + x = self._batch_norm(x) + x = self._relu(x) + orig_x = x + else: + orig_x = x + x = self._batch_norm(x) + x = self._relu(x) + + x = self._conv(x, 1, out_filter // 4, stride, is_atrous=True) + + x = self._batch_norm(x) + x = self._relu(x) + # pad when stride isn't unit + x = self._conv(x, 3, out_filter // 4, 1, is_atrous=True) + + x = self._batch_norm(x) + x = self._relu(x) + x = self._conv(x, 1, out_filter, 1, is_atrous=True) + + if in_filter != out_filter: + orig_x = self._conv(orig_x, 1, out_filter, stride, is_atrous=True) + x = tf.add(x, orig_x) + + tf.logging.info('image after unit %s: %s', name_scope, x.get_shape()) + return x + + def _conv(self, x, kernel_size, filters, strides, is_atrous=False): + """Convolution.""" + + padding = 'SAME' + if not is_atrous and strides > 1: + pad = kernel_size - 1 + pad_beg = pad // 2 + pad_end = pad - pad_beg + if self._data_format == 'channels_first': + x = tf.pad(x, [[0, 0], [0, 0], [pad_beg, pad_end], [pad_beg, pad_end]]) + else: + x = tf.pad(x, [[0, 0], [pad_beg, pad_end], [pad_beg, pad_end], [0, 0]]) + padding = 'VALID' + return tf.layers.conv2d( + inputs=x, + kernel_size=kernel_size, + filters=filters, + strides=strides, + padding=padding, + use_bias=False, + data_format=self._data_format) + + def _batch_norm(self, x): + if self._data_format == 'channels_first': + data_format = 'NCHW' + else: + data_format = 'NHWC' + return tf.contrib.layers.batch_norm( + x, + decay=self._batch_norm_decay, + center=True, + scale=True, + epsilon=self._batch_norm_epsilon, + is_training=self._is_training, + fused=True, + data_format=data_format) + + def _relu(self, x): + return tf.nn.relu(x) + + def _fully_connected(self, x, out_dim): + with tf.name_scope('fully_connected') as name_scope: + x = tf.layers.dense(x, out_dim) + + tf.logging.info('image after unit %s: %s', name_scope, x.get_shape()) + return x + + def _avg_pool(self, x, pool_size, stride): + with tf.name_scope('avg_pool') as name_scope: + x = tf.layers.average_pooling2d( + x, pool_size, stride, 'SAME', data_format=self._data_format) + + tf.logging.info('image after unit %s: %s', name_scope, x.get_shape()) + return x + + def _global_avg_pool(self, x): + with tf.name_scope('global_avg_pool') as name_scope: + assert x.get_shape().ndims == 4 + if self._data_format == 'channels_first': + x = tf.reduce_mean(x, [2, 3]) + else: + x = tf.reduce_mean(x, [1, 2]) + tf.logging.info('image after unit %s: %s', name_scope, x.get_shape()) + return x diff --git a/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/Dockerfile.gpu b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/Dockerfile.gpu new file mode 100644 index 0000000000000000000000000000000000000000..55aa1c9a1980543b609f7271635cb15fa9769919 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/Dockerfile.gpu @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nvidia/cuda:9.2-base-ubuntu18.04 + +RUN echo "$LOG_TAG update and install basic packages" && \ + apt-get -y update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + libfreetype6-dev \ + libpng-dev \ + libzmq3-dev \ + pkg-config \ + rsync \ + software-properties-common \ + unzip \ + vim \ + wget \ + && \ + apt-get install -y locales && \ + locale-gen $LANG && \ + apt-get clean && \ + apt -y autoclean && \ + apt -y dist-upgrade && \ + apt-get install -y build-essential && \ + rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +RUN echo "$LOG_TAG Install java8" && \ + apt-get -y update && \ + apt-get install -y openjdk-8-jdk && \ + rm -rf /var/lib/apt/lists/* + +# Install Zeppelin +ENV Z_VERSION="0.7.3" \ + Z_HOME="/zeppelin" + +RUN echo "$LOG_TAG Download Zeppelin binary" && \ + wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \ + tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \ + rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \ + mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME} +ENV PATH="${Z_HOME}/bin:${PATH}" + +RUN echo "$LOG_TAG Set locale" && \ + echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \ + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ + echo "LANG=en_US.UTF-8" > /etc/locale.conf && \ + locale-gen en_US.UTF-8 + +ENV LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + +COPY zeppelin-site.xml $Z_HOME/conf/zeppelin-site.xml +COPY shiro.ini ${Z_HOME}/conf/shiro.ini +RUN chmod 777 -R ${Z_HOME} + +COPY run_container.sh /usr/local/bin/run_container.sh +RUN chmod 755 /usr/local/bin/run_container.sh + +EXPOSE 8080 +CMD ["/usr/local/bin/run_container.sh"] diff --git a/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/run_container.sh b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/run_container.sh new file mode 100644 index 0000000000000000000000000000000000000000..8b909209915a24cdefb16da7f900eb707b9e8054 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/run_container.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"${Z_HOME}/bin/zeppelin-daemon.sh" start +while true; do + #perform the test + sleep 5 +done diff --git a/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/shiro.ini b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/shiro.ini new file mode 100644 index 0000000000000000000000000000000000000000..89f976af128cdbc2b8bc981c14a08d3a7e2392ec --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/shiro.ini @@ -0,0 +1,120 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[users] +# List of users with their password allowed to access Zeppelin. +# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections +# To enable admin user, uncomment the following line and set an appropriate password. +admin = admin, admin +user1 = password2, role1, role2 +user2 = password3, role3 +user3 = password4, role2 + +# Sample LDAP configuration, for user Authentication, currently tested for single Realm +[main] +### A sample for configuring Active Directory Realm +#activeDirectoryRealm = org.apache.zeppelin.realm.ActiveDirectoryGroupRealm +#activeDirectoryRealm.systemUsername = userNameA + +#use either systemPassword or hadoopSecurityCredentialPath, more details in http://zeppelin.apache.org/docs/latest/security/shiroauthentication.html +#activeDirectoryRealm.systemPassword = passwordA +#activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/zeppelin.jceks +#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM +#activeDirectoryRealm.url = ldap://ldap.test.com:389 +#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr" +#activeDirectoryRealm.authorizationCachingEnabled = false + +### A sample for configuring LDAP Directory Realm +#ldapRealm = org.apache.zeppelin.realm.LdapGroupRealm +## search base for ldap groups (only relevant for LdapGroupRealm): +#ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM +#ldapRealm.contextFactory.url = ldap://ldap.test.com:389 +#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM +#ldapRealm.contextFactory.authenticationMechanism = simple + +### A sample PAM configuration +#pamRealm=org.apache.zeppelin.realm.PamRealm +#pamRealm.service=sshd + +### A sample for configuring ZeppelinHub Realm +#zeppelinHubRealm = org.apache.zeppelin.realm.ZeppelinHubRealm +## Url of ZeppelinHub +#zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com +#securityManager.realms = $zeppelinHubRealm + +## A same for configuring Knox SSO Realm +#knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm +#knoxJwtRealm.providerUrl = https://domain.example.com/ +#knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html +#knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout +#knoxJwtRealm.logoutAPI = true +#knoxJwtRealm.redirectParam = originalUrl +#knoxJwtRealm.cookieName = hadoop-jwt +#knoxJwtRealm.publicKeyPath = /etc/zeppelin/conf/knox-sso.pem +# +#knoxJwtRealm.groupPrincipalMapping = group.principal.mapping +#knoxJwtRealm.principalMapping = principal.mapping +#authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter + +sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager + +### If caching of user is required then uncomment below lines +#cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager +#securityManager.cacheManager = $cacheManager + +### Enables 'HttpOnly' flag in Zeppelin cookies +cookie = org.apache.shiro.web.servlet.SimpleCookie +cookie.name = JSESSIONID +cookie.httpOnly = true +### Uncomment the below line only when Zeppelin is running over HTTPS +#cookie.secure = true +sessionManager.sessionIdCookie = $cookie + +securityManager.sessionManager = $sessionManager +# 86,400,000 milliseconds = 24 hour +securityManager.sessionManager.globalSessionTimeout = 86400000 +shiro.loginUrl = /api/login + +[roles] +role1 = * +role2 = * +role3 = * +admin = * + +[urls] +# This section is used for url-based security. For details see the shiro.ini documentation. +# +# You can secure interpreter, configuration and credential information by urls. +# Comment or uncomment the below urls that you want to hide: +# anon means the access is anonymous. +# authc means form based auth Security. +# +# IMPORTANT: Order matters: URL path expressions are evaluated against an incoming request +# in the order they are defined and the FIRST MATCH WINS. +# +# To allow anonymous access to all but the stated urls, +# uncomment the line second last line (/** = anon) and comment the last line (/** = authc) +# +/api/version = anon +# Allow all authenticated users to restart interpreters on a notebook page. +# Comment out the following line if you would like to authorize only admin users to restart interpreters. +/api/interpreter/setting/restart/** = authc +/api/interpreter/** = authc, roles[admin] +/api/configurations/** = authc, roles[admin] +/api/credential/** = authc, roles[admin] +#/** = anon +/** = authc diff --git a/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/zeppelin-site.xml b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/zeppelin-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..2bde161bd6a7882626a5d4ae495e8a424b2818c4 --- /dev/null +++ b/docs/userdocs/yarn/docker/tensorflow/zeppelin-notebook-example/zeppelin-site.xml @@ -0,0 +1,569 @@ + + + + + + + + zeppelin.server.addr + 0.0.0.0 + Server address + + + + zeppelin.server.port + 8080 + Server port. + + + + zeppelin.server.ssl.port + 8443 + Server ssl port. (used when ssl property is set to true) + + + + zeppelin.server.context.path + / + Context Path of the Web Application + + + + zeppelin.war.tempdir + webapps + Location of jetty temporary directory + + + + zeppelin.notebook.dir + notebook + path or URI for notebook persist + + + + zeppelin.notebook.homescreen + + id of notebook to be displayed in homescreen. ex) 2A94M5J1Z Empty value displays default home screen + + + + zeppelin.notebook.homescreen.hide + false + hide homescreen notebook from list when this value set to true + + + + zeppelin.notebook.collaborative.mode.enable + true + Enable collaborative mode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + zeppelin.notebook.storage + org.apache.zeppelin.notebook.repo.GitNotebookRepo + versioned notebook persistence layer implementation + + + + zeppelin.notebook.one.way.sync + false + If there are multiple notebook storages, should we treat the first one as the only source of truth? + + + + zeppelin.interpreter.dir + interpreter + Interpreter implementation base directory + + + + zeppelin.interpreter.localRepo + local-repo + Local repository for interpreter's additional dependency loading + + + + zeppelin.interpreter.dep.mvnRepo + http://repo1.maven.org/maven2/ + Remote principal repository for interpreter's additional dependency loading + + + + zeppelin.dep.localrepo + local-repo + Local repository for dependency loader + + + + zeppelin.helium.node.installer.url + https://nodejs.org/dist/ + Remote Node installer url for Helium dependency loader + + + + zeppelin.helium.npm.installer.url + http://registry.npmjs.org/ + Remote Npm installer url for Helium dependency loader + + + + zeppelin.helium.yarnpkg.installer.url + https://github.com/yarnpkg/yarn/releases/download/ + Remote Yarn package installer url for Helium dependency loader + + + + zeppelin.interpreters + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter + Comma separated interpreter configurations. First interpreter become a default + + + + zeppelin.interpreter.group.order + spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam,groovy + + + + + zeppelin.interpreter.connect.timeout + 30000 + Interpreter process connect timeout in msec. + + + + zeppelin.interpreter.output.limit + 102400 + Output message from interpreter exceeding the limit will be truncated + + + + zeppelin.ssl + false + Should SSL be used by the servers? + + + + zeppelin.ssl.client.auth + false + Should client authentication be used for SSL connections? + + + + zeppelin.ssl.keystore.path + keystore + Path to keystore relative to Zeppelin configuration directory + + + + zeppelin.ssl.keystore.type + JKS + The format of the given keystore (e.g. JKS or PKCS12) + + + + zeppelin.ssl.keystore.password + change me + Keystore password. Can be obfuscated by the Jetty Password tool + + + + + + zeppelin.ssl.truststore.path + truststore + Path to truststore relative to Zeppelin configuration directory. Defaults to the keystore path + + + + zeppelin.ssl.truststore.type + JKS + The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type + + + + + + zeppelin.server.allowed.origins + * + Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173 + + + + zeppelin.anonymous.allowed + false + Anonymous user allowed by default + + + + zeppelin.username.force.lowercase + false + Force convert username case to lower case, useful for Active Directory/LDAP. Default is not to change case + + + + zeppelin.notebook.default.owner.username + + Set owner role by default + + + + zeppelin.notebook.public + true + Make notebook public by default when created, private otherwise + + + + zeppelin.websocket.max.text.message.size + 1024000 + Size in characters of the maximum text message to be received by websocket. Defaults to 1024000 + + + + zeppelin.server.default.dir.allowed + false + Enable directory listings on server. + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/workbench/HowToRun.md b/docs/workbench/HowToRun.md new file mode 100644 index 0000000000000000000000000000000000000000..4ef2cb7835a8bbee886541ea16f5847307e0dc98 --- /dev/null +++ b/docs/workbench/HowToRun.md @@ -0,0 +1,114 @@ + +# How To Run Submarine Workbench +We provide two methods to launch Submarine Workbench +* Method 1: Run Submarine Workbench on docker +* Method 2: Run Submarine Workbench without docker + +# Run Submarine Workbench on docker + +By using the official images of Submarine, only a few docker commands are required to launch **Submarine Workbench**. The document includes information about how to launch the Submarine Workbench via the new docker images and the information about how to switch between different Submarine Workbench versions(version Vue & version Angular). + +### Two versions of Submarine Workbench +1. Angular (default) +2. Vue (This is the old version, and it will be replaced by version Angular in the future.) +#### (WARNING: Please restart a new **incognito window** when you switch to different versions of Submarine Workbench) +### Launch the Submarine Workbench(Angular) +* It should be noted that since Submarine Workbench depends on the Submarine database, so you need to run the docker container of the Submarine database first. +``` +docker run -it -p 3306:3306 -d --name submarine-database -e MYSQL_ROOT_PASSWORD=password apache/submarine:database- +docker run -it -p 8080:8080 -d --link=submarine-database:submarine-database --name submarine-server apache/submarine:server- +``` +* The login page of Submarine Workbench will be shown in ```http://127.0.0.1:8080```. +### Check the data in the submarine-database +* Step1: Enter the submarine-database container +``` +docker exec -it submarine-database bash +``` +* Step2: Enter MySQL database +``` +mysql -uroot -ppassword +``` +* Step3: List the data in the table +``` +// list all databases +show databases; + +// choose a database +use ${target_database}; + +// list all tables +show tables; + +// list the data in the table +select * from ${target_table}; +``` +# Run Submarine Workbench without docker +### Run Submarine Workbench + +``` +cd submarine +./bin/submarine-daemon.sh [start|stop|restart] +``` +To start workbench server, you need to download MySQL jdbc jar and put it in the +path of workbench/lib for the first time. Or you can add parameter, getMysqlJar, +to get MySQL jar automatically. +``` +cd submarine +./bin/submarine-daemon.sh start getMysqlJar +``` + +### submarine-env.sh + +`submarine-env.sh` is automatically executed each time the `submarine-daemon.sh` script is executed, so we can set the `submarine-daemon.sh` script and the environment variables in the `SubmarineServer` process via `submarine-env.sh`. + +| Name | Variable | +| ------------------- | ------------------------------------------------------------ | +| JAVA_HOME | Set your java home path, default is `java`. | +| SUBMARINE_JAVA_OPTS | Set the JAVA OPTS parameter when the Submarine Workbench process starts. If you need to debug the Submarine Workbench process, you can set it to `-agentlib:jdwp=transport=dt_socket, server=y,suspend=n,address=5005` | +| SUBMARINE_MEM | Set the java memory parameter when the Submarine Workbench process starts. | +| MYSQL_JAR_URL | The customized URL to download MySQL jdbc jar. | +| MYSQL_VERSION | The version of MySQL jdbc jar to downloaded. The default value is 5.1.39. It's used to generate the default value of MYSQL_JDBC_URL | + +### submarine-site.xml + +`submarine-site.xml` is the configuration file for the entire `Submarine` system to run. + +| Name | Variable | +| ---------------------------------- | ------------------------------------------------------------ | +| submarine.server.addr | Submarine server address, default is `0.0.0.0` | +| submarine.server.port | Submarine server port, default `8080` | +| submarine.ssl | Should SSL be used by the Submarine servers?, default `false` | +| submarine.server.ssl.port | Server ssl port. (used when ssl property is set to true), default `8483` | +| submarine.ssl.client.auth | Should client authentication be used for SSL connections? | +| submarine.ssl.keystore.path | Path to keystore relative to Submarine configuration directory | +| submarine.ssl.keystore.type | The format of the given keystore (e.g. JKS or PKCS12) | +| submarine.ssl.keystore.password | Keystore password. Can be obfuscated by the Jetty Password tool | +| submarine.ssl.key.manager.password | Key Manager password. Defaults to keystore password. Can be obfuscated. | +| submarine.ssl.truststore.path | Path to truststore relative to Submarine configuration directory. Defaults to the keystore path | +| submarine.ssl.truststore.type | The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type | +| submarine.ssl.truststore.password | Truststore password. Can be obfuscated by the Jetty Password tool. Defaults to the keystore password | +| workbench.web.war | Submarine Workbench web war file path. | + + + +### Compile + +[Build From Code Guide](../development/BuildFromCode.md) + +```$xslt +cd submarine/submarine-dist/target/submarine-dist-/submarine-dist-/ +./bin/submarine-daemon.sh [start|stop|restart] +``` diff --git a/docs/workbench/README.md b/docs/workbench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..77a3379a67ee3bbae7abd0449ad87ae365a925d7 --- /dev/null +++ b/docs/workbench/README.md @@ -0,0 +1,177 @@ + + +English | [简体中文](./README.zh-CN.md) + +# Submarine Workbench Introduction + +Submarine Workbench is a WEB system for data scientists. +Data scientists can interactively access notebooks, submit/manage jobs, manage models, create model training workflows, access data sets, and more through Submarine Workbench. + +## Register + +Everyone who needs to use Submarine for machine learning algorithm development can log in to Submarine Workbench's WEB homepage. On the homepage, click the registration link, fill in the user name, email address and password to register the user. At this time, the user status is `waiting for approval` status. + +After receiving the registration request from the user in Submarine Workbench, the administrator sets the operation authority according to the user's needs, sets the user's organization and allocates resources, and sets the user status to `pass the audit`. The user can log in to the Submarine Workbench. Different users have different permission. + +## Login + +Each Submarine user logs in to the Home page of Submarine Workbench by entering their username and password on the Login page. + +## Home + +In the Submarine Workbench Home page, the top level shows the user's resource usage and task execution through four charts. + +In the `Quick Start` list, the most commonly used feature links in the Workbench are displayed so that users can work quickly. + +In the `Open Recent` list, there are nine items that the user has used recently, so you can work quickly. + +At `What's New?` In the list, some of the latest features and project information released by Submarine are displayed to help you understand the latest developments in the Submarine project. + +## Workspace + +Workspace consists primarily of five tab pages, with the total number of items in each tab page's title. + +### Project + +In the Project page, all the projects created by the user themselves are displayed as cards. + +![image-20191007161424534](assets/workspace-project.png) + +Each Project card consists of the following sections: + +1. **Project Type**：Submarine currently supports six types of machine learning algorithm frameworks and development languages: `Notebook`, `Python`, `R`, `Scala`, `Tensorflow`, and `PyTorch`, which are identified by corresponding icons in the project card. +2. **Project Tags**：Users can tag each Project with different tags for easy searching and management. +3. **Github/Gitlab integrated**：Submarine Workbench is system integrated with `Github`/`Gitlab`, and each Project can perform `Watch`, `Star`, `Fork`, and `Comment `operations in Workbench. + + **Watch**：[TODO] + + **Star**：[TODO] + + **Fork**：[TODO] + + **Comment**：Users can comment on the project. +4. **Edit**：Users can open projects in **Notebook** and perform algorithm development by double-clicking on the project or by clicking the **Edit** button. +5. **Download**：The user downloads the project package locally by clicking the **Download** button. +6. **Setting**：Edit project information such as project name, profile, visibility level and permissions. +7. **Delete**：Delete the project and all included files. + +#### Add New Project + +Clicking the **Add New Project** button on the project page will display the guide page for creating the project, and you can create a new project in just three steps. + +**Step 1**: Fill in the project name and project description in the **Base Information** step. + +![image-20191007171638338](assets/workspace-project-step1.png) + ++ **Visibility**: Set the visibility level of the item externally + + + **Private**: (Default) Set to private project, and all the files included in the project are not publicly displayed. but the execution result of the project can be individually set and exposed in Notebook, so that others can view the visual report of the project. + + **Team**: Set to team project, select the team name in the team selection box, and other members of the team can access the project according to the set permissions. + + **Public**: Set to public project, all users in Workbench can view this project through search. ++ **Permission**: Set the external access rights of the project. The permission setting interface will appear only when the **Visibility** of the project is set to **Team** or **Public**. + + + **Can View** + + When the project's **Visibility** is set to **Team**, other members of the team can **only view** the files for this project. + + When the project's **Visibility** is set to **Public**, other members of the Workbench can **only view** the files for this project. + + + **Can Edit** + + When the project's **Visibility** is set to **Team**, other members of the team can **view** and **edit** the files for this project. + + When the project's **Visibility** is set to **Public**, other members of the Workbench can **view** and **edit** the files for this project. + + + **Can Execute** + + When the project's **Visibility** is set to **Team**, other members of the team can **view**, **edit**, and **execute** the project's files. + + When the project's **Visibility** is set to **Public**, other members of the Workbench can **view**, **edit**, and **execute** the project's files. + +**Step 2**: In the Initial Project step, Workbench provides four ways to initialize the project. + ++ **Template**: Workbench Project templates with several different development languages and algorithm frameworks are built in. You can choose any template to initialize your project and you can execute it directly in Notebook without any modification. It is especially suitable for novices to experience quickly. + + ![image-20191007184749193](assets/workspace-project-step2-template.png) + ++ **Blank**：Create a blank project, and later we can manually add the project's file in Notebook + + ![image-20191007184811389](assets/workspace-project-step2-blank.png) + ++ **Upload**: Initialize your project by uploading a file in notebook format that is compatible with the **Jupyter Notebook** and **Zeppelin Notebook** file formats. + + ![image-20191007184825531](assets/workspace-project-step2-upload.png) + ++ **Git Repo**: Fork a file in the repository to initialize the project in your **Github**/**Gitlab** account. + + ![image-20191007184840989](assets/workspace-project-step2-git.png) + +**Step 3**：Preview the included files in the project + +![image-20191007191205660](assets/workspace-project-step3.png) + ++ **Save**: Save the project to Workspace. ++ **Open In Notebook**: Save the project to Workspace and open the project with Notebook. + +### Release + +[TODO] + +### Training + +[TODO] + +### Team + +[TODO] + +### Shared + +[TODO] + +## Interpreters + +[TODO] + +## Job + +[TODO] + +## Data + +[TODO] + +## Model + +[TODO] + +## Manager + +### User + +[TODO] + +### Team + +[TODO] + +### Data Dict + +[TODO] + +### Department + +[TODO] + +## How to run workbench + +[How To Run Submarine Workbench Guide](./HowToRun.md) \ No newline at end of file diff --git a/docs/workbench/README.zh-CN.md b/docs/workbench/README.zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..edebaf92284443e9055fdc0ac5e9f230aef45395 --- /dev/null +++ b/docs/workbench/README.zh-CN.md @@ -0,0 +1,176 @@ + + +[English](./README.md) | 简体中文 + +# Submarine Workbench Introduction + +`Submarine Workbench` 是为数据科学家设计的 WEB 系统。数据科学家可以通过 `Submarine Workbench` 进行交互式的访问 `Notebook`，提交/管理 Job，管理模型，创建模型训练工作流，访问数据集等。 + +## Register + +每个需要使用 Submarine 进行机器学习算法开发的用户，都可以登录 `Submarine Workbench` 的 WEB 首页，在首页上，点击注册链接，填写用户名、注册邮箱和密码就可以完成注册，但此时用户状态为 `等待审核` 状态。 + +管理员在 `Submarine Workbench` 中接收到用户的注册请求后，设置用户的操作权限，所属机构部门和分配资源，设置用户状态为 `审核通过` 后，用户才可以登录 Submarine Workbench。 + +## Login + +每个 Submarine 的用户在 `Login` 页面中输入用户名和密码，登录到 `Submarine Workbench` 的首页 `Home`。 + +## Home + +在 `Submarine Workbench` 的 `Home` 首页中，顶层通过四个图表显示了用户的资源的使用情况和任务执行的情况。 + +在 `Quick Start` 列表中，显示了 Workbench 中最常使用的功能链接，方便用户可以快速的进行工作。 + +在 `Open Recent` 列表中，显示了用户最近使用过的九个项目，方便你快速的进行工作。 + +在 `What‘s New？` 列表中，显示了 Submarine 最新发布的一些功能特性和项目信息，方便你了解 Submarine 项目的最新进展。 + +## Workspace + +Workspace 主要有五个 Tab 页组成，每个 Tab 页的标题中显示了各自项目的总数。 + +### Project + +在 Project 页面中，以卡片的方式显示了用户自己创建的所有 Project。 + +![image-20191007161424534](assets/workspace-project.png) + +每个 Project 卡片由以下部分内容组成： + +1. **Project 类型**：目前 Submarine 支持 `Notebook`、`Python`、`R`、`Scala`、`Tensorflow` 和 `PyTorch` 这六种类型的机器学习算法框架和开发语言，在项目卡片中以对应的图标进行标识。 +2. **Project Tags**：用户可以为每个 Project 打上不同的 `Tag` 标签，方便查找和管理。 +3. **Github/Gitlab 集成**：Submarine Workbench 与 `Github`/`Gitlab` 进行了系统集成，每个 Project 都可以在 Workbench 中进行 `Watch`、`Star`、`Frok` 和 `Comment` 操作。 + + **Watch**：[TODO] + + **Star**：[TODO] + + **Fork**：[TODO] + + **Comment**：用户可以在项目中进行评论 +4. **Edit**：用户通过双击项目或者点击 `Edit` 按钮，可以在 `Notebook` 中打开项目，进行算法开发等操作。 +5. **Download**：用户通过点击 `Download` 按钮，将项目打包下载到本地。 +6. **Setting**：编辑项目信息，例如项目的名字，简介，分享级别和权限。 +7. **Delete**：删除项目中所有包含的文件。 + +#### Add New Project + +在项目页面中点击 `Add New Project` 按钮，将会显示出创建项目的引导页面，只需要三个步骤就可以创建一个新的项目。 + +第一步：在 **Base Information** 步骤中填写项目名称、项目简介。 + +![image-20191007171638338](assets/workspace-project-step1.png) + ++ **Visibility**: 设置项目对外的可见级别 + + + **Private**: （默认）设置为私有项目，不对外公开项目中包含的所有文件，但是可以在 **Notebook** 中将项目的执行结果单独设置公开，方便其他人查看项目的可视化报告。 + + **Team**: 设置为团队项目，在团队选择框中选择团队的名称，团队的其他成员可以根据设置的权限访问这个项目。 + + **Public**: 设置为公开项目，**Workbench** 中的所有用户都可以通过搜索查看到这个项目。 ++ **Permission**: 设置项目对外的访问权限，只有将项目的 **Visibility** 设置为 **Team** 或 **Public** 的时候，才会出现权限设置界面。 + + + **Can View** + + 当项目的 **Visibility** 设置为 **Team** 时，团队中其他成员都只能**查看**这个项目的文件。 + + 当项目的 Visibility 设置为 **Public** 时，**Workbench** 中其他成员都只能**查看**这个项目的文件。 + + + **Can Edit** + + 当项目的 **Visibility** 设置为 **Team** 时，团队中其他成员都可以**查看**、**编辑**这个项目的文件。 + + 当项目的 **Visibility** 设置为 **Public** 时，**Workbench** 中其他成员都可以**查看**、**编辑**这个项目的文件。 + + + **Can Execute** + + 当项目的 **Visibility** 设置为 **Team** 时，团队中其他成员都可以**查看**、**编辑**、**执行**这个项目的文件。 + + 当项目的 **Visibility** 设置为 **Public** 时，**Workbench** 中其他成员都可以**查看**、**编辑**、**执行**这个项目的文件。 + +第二步：在 **Initial Project** 步骤中，**Workbench** 提供了四种项目初始化的方式 + ++ **Template**: **Workbench** 内置了几种不同开发语言和算法框架的项目模版，你可以选择任何一种模版初始化你的项目，无需做任何修改就可以直接在 **Notebook** 中执行，特别适合新手进行快速的体验。 + + ![image-20191007184749193](assets/workspace-project-step2-template.png) + ++ **Blank**：创建一个空白的项目，稍后，我们可以通过在 **Notebook** 中手工添加项目的文件 + + ![image-20191007184811389](assets/workspace-project-step2-blank.png) + ++ **Upload**: 通过上传 **notebook** 格式的文件来初始化你的项目，**notebook** 格式兼容 **Jupyter Notebook** 和 **Zeppelin Notebook** 文件格式。 + + ![image-20191007184825531](assets/workspace-project-step2-upload.png) + ++ **Git Repo**: 在你的 **Github**/**Gitlab** 账号中 **Fork** 一个仓库中的文件内容来初始化项目。 + + ![image-20191007184840989](assets/workspace-project-step2-git.png) + +第三步：预览项目中的所包含的文件 + +![image-20191007191205660](assets/workspace-project-step3.png) + ++ **Save**: 将项目保存到 Workspace 中。 ++ **Open In Notebook**: 将项目保存到 **Workspace** 中，并用 **Notebook** 打开项目。 + +### Release + +[TODO] + +### Training + +[TODO] + +### Team + +[TODO] + +### Shared + +[TODO] + +## Interpreters + +[TODO] + +## Job + +[TODO] + +## Data + +[TODO] + +## Model + +[TODO] + +## Manager + +### User + +[TODO] + +### Team + +[TODO] + +### Data Dict + +[TODO] + +### Department + +[TODO] + +## How to run workbench + +[How To Run Submarine Workbench Guide](./HowToRun.md) \ No newline at end of file diff --git a/docs/workbench/assets/workspace-project-step1.png b/docs/workbench/assets/workspace-project-step1.png new file mode 100644 index 0000000000000000000000000000000000000000..c090e7cf95e1de6e44eb8fd18a559966c6083240 Binary files /dev/null and b/docs/workbench/assets/workspace-project-step1.png differ diff --git a/docs/workbench/assets/workspace-project-step2-blank.png b/docs/workbench/assets/workspace-project-step2-blank.png new file mode 100644 index 0000000000000000000000000000000000000000..e6a09548c6154c61b60c59fd9009ea8c256d6216 Binary files /dev/null and b/docs/workbench/assets/workspace-project-step2-blank.png differ diff --git a/docs/workbench/assets/workspace-project-step2-git.png b/docs/workbench/assets/workspace-project-step2-git.png new file mode 100644 index 0000000000000000000000000000000000000000..cf4730e8f84d175bc19708bacac5ff59a365337e Binary files /dev/null and b/docs/workbench/assets/workspace-project-step2-git.png differ diff --git a/docs/workbench/assets/workspace-project-step2-template.png b/docs/workbench/assets/workspace-project-step2-template.png new file mode 100644 index 0000000000000000000000000000000000000000..f28d4cf91ff445773e09b8f5e94fd5a01b81c09f Binary files /dev/null and b/docs/workbench/assets/workspace-project-step2-template.png differ diff --git a/docs/workbench/assets/workspace-project-step2-upload.png b/docs/workbench/assets/workspace-project-step2-upload.png new file mode 100644 index 0000000000000000000000000000000000000000..c70bad14ef6f5e9e545e0e23a7c20c729c8de8cd Binary files /dev/null and b/docs/workbench/assets/workspace-project-step2-upload.png differ diff --git a/docs/workbench/assets/workspace-project-step3.png b/docs/workbench/assets/workspace-project-step3.png new file mode 100644 index 0000000000000000000000000000000000000000..f7be3840f7aa6c97383d722d0df2701be02eb3c6 Binary files /dev/null and b/docs/workbench/assets/workspace-project-step3.png differ diff --git a/docs/workbench/assets/workspace-project.png b/docs/workbench/assets/workspace-project.png new file mode 100644 index 0000000000000000000000000000000000000000..1e8cefab005599bd3d5324cfbca1de36516ae630 Binary files /dev/null and b/docs/workbench/assets/workspace-project.png differ diff --git a/docs/workbench/notebook/jupyter.yaml b/docs/workbench/notebook/jupyter.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32d065ebf4f50bf7f70b1d07b3f741ce690897eb --- /dev/null +++ b/docs/workbench/notebook/jupyter.yaml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: jupyter-deployment + labels: + app: jupyter +spec: + selector: + matchLabels: + app: jupyter + replicas: 1 + template: + metadata: + labels: + namespace: submarine + app: jupyter + spec: + nodeSelector: + kubernetes.io/hostname: k8s-submarine-control-plane + containers: + - name: jupyter + image: jupyter/minimal-notebook + # mount path in jupyter container + volumeMounts: + - mountPath: /home/jovyan + name: jupyter-storage + volumes: + - name: jupyter-storage + # this path must be the same as extraMounts.containerPath in the kind config + hostPath: + path: /tmp/submarine/ + type: DirectoryOrCreate +--- +kind: Service +apiVersion: v1 +metadata: + name: jupyter-service +spec: + selector: + app: jupyter + type: NodePort + ports: + - port: 8888 + targetPort: 8888 + nodePort: 30070 diff --git a/docs/workbench/notebook/setup-jupyter.md b/docs/workbench/notebook/setup-jupyter.md new file mode 100644 index 0000000000000000000000000000000000000000..e15ffdc518fc4cfc6686b58828b3070ef130b83a --- /dev/null +++ b/docs/workbench/notebook/setup-jupyter.md @@ -0,0 +1,73 @@ + +# Deploy Jupyter Notebook on Kubernetes +This guide covers the deployment Jupyter Notebook on kubernetes cluster. + +## Experiment environment +### Setup Kubernetes +We recommend using [kind](https://kind.sigs.k8s.io/) to setup a Kubernetes cluster on a local machine. + +You can use Extra mounts to mount your host path to kind node and use Extra port mappings to port +forward to the kind nodes. Please refer to [kind configuration](https://kind.sigs.k8s.io/docs/user/configuration/#extra-mounts) +for more details. + +You need to create a kind config file. The following is an example : +``` +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraMounts: + # add a mount from /path/to/my/files on the host to /files on the node + - hostPath: /tmp/submarine + containerPath: /tmp/submarine + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + # exposing additional ports to be used for NodePort services + - containerPort: 30070 + hostPort: 8888 + protocol: TCP +``` + +Running the following command: + +``` +kind create cluster --image kindest/node:v1.15.6 --config --name k8s-submarine +kubectl create namespace submarine +``` + +### Deploy Jupyter Notebook +Once you have a running Kubernetes cluster, you can write a YAML file to deploy a jupyter notebook. +In this [example yaml](./jupyter.yaml), we use [jupyter/minimal-notebook](https://hub.docker.com/r/jupyter/minimal-notebook/) +to make a single notebook running on the kind node. + +``` +kubectl apply -f jupyter.yaml --namespace submarine +``` + +Once jupyter notebook is running, you can access the notebook server from the browser using http://localhost:8888 on local machine. + +You can enter and store a password for your notebook server with: +``` +kubectl exec -it -- jupyter notebook password +``` +After restarting the notebook server, you can login jupyter notebook with your new password. + +If you want to use JupyterLab : +``` +http://localhost:8888/lab +``` diff --git a/helm-charts/submarine/.helmignore b/helm-charts/submarine/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..b92c4a41bd2ddb33c3dbccaf7629f89f7b4e6359 --- /dev/null +++ b/helm-charts/submarine/.helmignore @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# ignore dolder +ignore/* + diff --git a/helm-charts/submarine/Chart.yaml b/helm-charts/submarine/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f7c49b513791e3092c2475a8a421bf906593455 --- /dev/null +++ b/helm-charts/submarine/Chart.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v2 +appVersion: "0.5.0" +description: Submarine is Cloud Native Machine Learning Platform. +name: submarine +version: 0.5.0 +icon: https://submarine.apache.org/assets/themes/submarine/img/submarine_white_logo.png +dependencies: +- name: tfjob + version: "0.1.0" +- name: pytorchjob + version: "0.1.0" +- name: notebook-controller + version: "0.1.0" +- name: traefik + version: "9.1.0" + condition: submarine.traefik.enabled diff --git a/helm-charts/submarine/charts/notebook-controller/.helmignore b/helm-charts/submarine/charts/notebook-controller/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..7887c2c7ca020d4c72d690bd21908af05789e2f6 --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/.helmignore @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-charts/submarine/charts/notebook-controller/Chart.yaml b/helm-charts/submarine/charts/notebook-controller/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7024f34c7f5d0f24160e722d361735eb5157683 --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/Chart.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v2 +name: notebook-controller +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.0.0 diff --git a/helm-charts/submarine/charts/notebook-controller/crds/crd.yaml b/helm-charts/submarine/charts/notebook-controller/crds/crd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..390f5c7638dfbcfed2848b32672b843bd1202701 --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/crds/crd.yaml @@ -0,0 +1,80 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: notebooks.kubeflow.org +spec: + group: kubeflow.org + names: + kind: Notebook + plural: notebooks + singular: notebook + scope: Namespaced + subresources: + status: {} + versions: + - name: v1alpha1 + served: true + storage: false + - name: v1beta1 + served: true + storage: true + - name: v1 + served: true + storage: false + validation: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + template: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + properties: + spec: + type: object + type: object + type: object + status: + properties: + conditions: + description: Conditions is an array of current conditions + items: + properties: + type: + description: Type of the confition/ + type: string + required: + - type + type: object + type: array + required: + - conditions + type: object diff --git a/helm-charts/submarine/charts/notebook-controller/templates/cluster-role-binding.yaml b/helm-charts/submarine/charts/notebook-controller/templates/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e4180f48ec2ad54456fdca54e20f806574d3aed --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/templates/cluster-role-binding.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: notebook-controller-role-binding + labels: + app: notebook-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: notebook-controller-role +subjects: +- kind: ServiceAccount + name: notebook-controller-service-account + namespace: {{ .Release.Namespace }} diff --git a/helm-charts/submarine/charts/notebook-controller/templates/cluster-role.yaml b/helm-charts/submarine/charts/notebook-controller/templates/cluster-role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf7e9768e7e0abdf880e9a3967895e2619eb1a60 --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/templates/cluster-role.yaml @@ -0,0 +1,128 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: notebook-controller-role + labels: + app: notebook-controller +rules: +- apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - '*' +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - kubeflow.org + resources: + - notebooks + - notebooks/status + - notebooks/finalizers + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - virtualservices + verbs: + - '*' + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-notebooks-admin + labels: + app: notebook-controller + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true" +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-notebooks-admin: "true" +rules: [] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-notebooks-edit + labels: + app: notebook-controller + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-notebooks-admin: "true" +rules: +- apiGroups: + - kubeflow.org + resources: + - notebooks + - notebooks/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-notebooks-view + labels: + app: notebook-controller + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" +rules: +- apiGroups: + - kubeflow.org + resources: + - notebooks + - notebooks/status + verbs: + - get + - list + - watch diff --git a/helm-charts/submarine/charts/notebook-controller/templates/deployment.yaml b/helm-charts/submarine/charts/notebook-controller/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b053733a6825a516d45543954f798a4692ee81c --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/templates/deployment.yaml @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: notebook-controller-deployment + labels: + app: notebook-controller +spec: + selector: + matchLabels: + app: notebook-controller + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + labels: + app: notebook-controller + spec: + containers: + - name: manager + image: apache/submarine:notebook-controller-v1.1.0-g253890cb + command: + - /manager + imagePullPolicy: IfNotPresent + env: + - name: USE_ISTIO + value: "false" + livenessProbe: + httpGet: + path: /metrics + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 30 + serviceAccountName: notebook-controller-service-account diff --git a/helm-charts/submarine/charts/notebook-controller/templates/service-account.yaml b/helm-charts/submarine/charts/notebook-controller/templates/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54aa3594619c3590ed8df45ac8e57994cc46ae3e --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/templates/service-account.yaml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ServiceAccount +metadata: + name: notebook-controller-service-account + labels: + app: notebook-controller diff --git a/helm-charts/submarine/charts/notebook-controller/templates/service.yaml b/helm-charts/submarine/charts/notebook-controller/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d133a68ab7d690022c0e3173f70e8defbd231e2b --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/templates/service.yaml @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: Service +metadata: + name: notebook-controller-service + labels: + app: notebook-controller +spec: + type: ClusterIP + selector: + app: notebook-controller + ports: + - port: 443 diff --git a/helm-charts/submarine/charts/notebook-controller/values.yaml b/helm-charts/submarine/charts/notebook-controller/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cce3acad34a49511f09625f0f965df4ab3ea9e66 --- /dev/null +++ b/helm-charts/submarine/charts/notebook-controller/values.yaml @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/helm-charts/submarine/charts/pytorchjob/.helmignore b/helm-charts/submarine/charts/pytorchjob/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..cab781adbff7707de12a30199935f60ec114f4c8 --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/.helmignore @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-charts/submarine/charts/pytorchjob/Chart.yaml b/helm-charts/submarine/charts/pytorchjob/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c4b80bde98b85e7155e70e44173dc689106c94c --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/Chart.yaml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v2 +name: pytorchjob +description: pytorchjob + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.0.0 diff --git a/helm-charts/submarine/charts/pytorchjob/crds/crd.yaml b/helm-charts/submarine/charts/pytorchjob/crds/crd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5804ccd1b273af1233b0e8539386c12020951efc --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/crds/crd.yaml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pytorchjobs.kubeflow.org +spec: + additionalPrinterColumns: + - JSONPath: .status.conditions[-1:].type + name: State + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: kubeflow.org + names: + kind: PyTorchJob + plural: pytorchjobs + singular: pytorchjob + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + pytorchReplicaSpecs: + properties: + Master: + properties: + replicas: + maximum: 1 + minimum: 1 + type: integer + Worker: + properties: + replicas: + minimum: 1 + type: integer + versions: + - name: v1 + served: true + storage: true diff --git a/helm-charts/submarine/charts/pytorchjob/templates/deployment.yaml b/helm-charts/submarine/charts/pytorchjob/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c46fe00b4fa016fc835868393dd8710ac4dd4a7 --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/templates/deployment.yaml @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pytorch-operator +spec: + replicas: 1 + selector: + matchLabels: + name: pytorch-operator + template: + metadata: + labels: + name: pytorch-operator + spec: + containers: + - command: + - /pytorch-operator.v1 + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: apache/submarine:pytorch-operator-v1.1.0-gd596e904 + name: pytorch-operator + serviceAccountName: pytorch-operator diff --git a/helm-charts/submarine/charts/pytorchjob/templates/podgroup.yaml b/helm-charts/submarine/charts/pytorchjob/templates/podgroup.yaml new file mode 100644 index 0000000000000000000000000000000000000000..109d9a332890cef44737d539b29d754f85034d64 --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/templates/podgroup.yaml @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: podgroups.scheduling.incubator.k8s.io +spec: + group: scheduling.incubator.k8s.io + names: + kind: PodGroup + plural: podgroups + scope: Namespaced + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + minMember: + format: int32 + type: integer + type: object + status: + properties: + succeeded: + format: int32 + type: integer + failed: + format: int32 + type: integer + running: + format: int32 + type: integer + type: object + type: object + version: v1alpha1 diff --git a/helm-charts/submarine/charts/pytorchjob/templates/rbac.yaml b/helm-charts/submarine/charts/pytorchjob/templates/rbac.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09170407add40318827d14823f857fbb66a10263 --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/templates/rbac.yaml @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: pytorch-operator + name: pytorch-operator +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + app: pytorch-operator + name: pytorch-operator +rules: +- apiGroups: + - kubeflow.org + resources: + - pytorchjobs + - pytorchjobs/status + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - events + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + app: pytorch-operator + name: pytorch-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pytorch-operator +subjects: +- kind: ServiceAccount + name: pytorch-operator + namespace: {{ .Release.Namespace }} +--- diff --git a/helm-charts/submarine/charts/pytorchjob/templates/service.yaml b/helm-charts/submarine/charts/pytorchjob/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0580f0eeb0fdb175d748063d5018ab4d442c8b7 --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/templates/service.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "8443" + prometheus.io/scrape: "true" + labels: + app: pytorch-operator + name: pytorch-operator +spec: + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: + name: pytorch-operator + type: ClusterIP diff --git a/helm-charts/submarine/charts/pytorchjob/values.yaml b/helm-charts/submarine/charts/pytorchjob/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..555eccfe8542c37bc286cc412b84927c040383ab --- /dev/null +++ b/helm-charts/submarine/charts/pytorchjob/values.yaml @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + diff --git a/helm-charts/submarine/charts/tfjob/.helmignore b/helm-charts/submarine/charts/tfjob/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..cab781adbff7707de12a30199935f60ec114f4c8 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/.helmignore @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-charts/submarine/charts/tfjob/Chart.yaml b/helm-charts/submarine/charts/tfjob/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adffae8e1516ebe39c045ddd75819fd9427057c0 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/Chart.yaml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v2 +name: tfjob +description: tfjob + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.0.0 diff --git a/helm-charts/submarine/charts/tfjob/crds/crd.yaml b/helm-charts/submarine/charts/tfjob/crds/crd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1064757b283c383d54d039d5d9109f20a0bf81f5 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/crds/crd.yaml @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tfjobs.kubeflow.org +spec: + additionalPrinterColumns: + - JSONPath: .status.conditions[-1:].type + name: State + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: kubeflow.org + names: + kind: TFJob + plural: tfjobs + singular: tfjob + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + tfReplicaSpecs: + properties: + Chief: + properties: + replicas: + maximum: 1 + minimum: 1 + type: integer + PS: + properties: + replicas: + minimum: 1 + type: integer + Worker: + properties: + replicas: + minimum: 1 + type: integer + versions: + - name: v1 + served: true + storage: true diff --git a/helm-charts/submarine/charts/tfjob/templates/cluster-role-binding.yaml b/helm-charts/submarine/charts/tfjob/templates/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f148398fa678e4f192abea5389aa993d1e63834 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/templates/cluster-role-binding.yaml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + app: tf-job-operator + name: tf-job-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tf-job-operator +subjects: +- kind: ServiceAccount + name: tf-job-operator + namespace: {{ .Release.Namespace }} diff --git a/helm-charts/submarine/charts/tfjob/templates/cluster-role.yaml b/helm-charts/submarine/charts/tfjob/templates/cluster-role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2620ca9d7a78262b75b48d51ee8decb6e18950f8 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/templates/cluster-role.yaml @@ -0,0 +1,111 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + app: tf-job-operator + name: tf-job-operator +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + - tfjobs/status + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - events + verbs: + - '*' +- apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - '*' + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-tfjobs-admin + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true" +aggregationRule: + clusterRoleSelectors: + - matchLabels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-tfjobs-admin: "true" +rules: [] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-tfjobs-edit + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-tfjobs-admin: "true" +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + - tfjobs/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflow-tfjobs-view + labels: + rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + - tfjobs/status + verbs: + - get + - list + - watch diff --git a/helm-charts/submarine/charts/tfjob/templates/deployment.yaml b/helm-charts/submarine/charts/tfjob/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..880cb27a3f4a07af036108e126568fcb7f0a4f45 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/templates/deployment.yaml @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tf-job-operator + labels: + app: tf-job-operator + name: tf-job-operator +spec: + replicas: 1 + selector: + matchLabels: + app: tf-job-operator + name: tf-job-operator + template: + metadata: + labels: + app: tf-job-operator + name: tf-job-operator + spec: + containers: + - args: + - --alsologtostderr + - -v=1 + - --monitoring-port=8443 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: apache/submarine:tf_operator-v1.1.0-g92389064 + name: tf-job-operator + serviceAccountName: tf-job-operator diff --git a/helm-charts/submarine/charts/tfjob/templates/service-account.yaml b/helm-charts/submarine/charts/tfjob/templates/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..217e05b9584e9b2ccf025563368573ea47cd72d0 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/templates/service-account.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: tf-job-dashboard + name: tf-job-dashboard +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: tf-job-operator + name: tf-job-operator diff --git a/helm-charts/submarine/charts/tfjob/templates/service.yaml b/helm-charts/submarine/charts/tfjob/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e84ad6d2abd2dd489644a74199e4db337c0daa24 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/templates/service.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/scrape: "true" + prometheus.io/port: "8443" + labels: + app: tf-job-operator + name: tf-job-operator +spec: + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: + name: tf-job-operator + type: ClusterIP diff --git a/helm-charts/submarine/charts/tfjob/values.yaml b/helm-charts/submarine/charts/tfjob/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cce3acad34a49511f09625f0f965df4ab3ea9e66 --- /dev/null +++ b/helm-charts/submarine/charts/tfjob/values.yaml @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/helm-charts/submarine/charts/traefik/.helmignore b/helm-charts/submarine/charts/traefik/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..cab781adbff7707de12a30199935f60ec114f4c8 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/.helmignore @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm-charts/submarine/charts/traefik/Chart.yaml b/helm-charts/submarine/charts/traefik/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ec6bc56329febfebd831b27d0584623c15ec796 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/Chart.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v2 +name: traefik +description: A Traefik based Kubernetes ingress controller +type: application +version: 9.1.0 +appVersion: 2.2.8 +keywords: + - traefik + - ingress +home: https://traefik.io/ +sources: + - https://github.com/containous/traefik +icon: https://raw.githubusercontent.com/containous/traefik/v2.2/docs/content/assets/img/traefik.logo.png diff --git a/helm-charts/submarine/charts/traefik/crds/ingressroute-tcp.yaml b/helm-charts/submarine/charts/traefik/crds/ingressroute-tcp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dbaceaedfe84967ef4aef5d024561d2ce24cb8a --- /dev/null +++ b/helm-charts/submarine/charts/traefik/crds/ingressroute-tcp.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutetcps.traefik.containo.us +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRouteTCP + plural: ingressroutetcps + singular: ingressroutetcp + scope: Namespaced diff --git a/helm-charts/submarine/charts/traefik/crds/ingressroute-udp.yaml b/helm-charts/submarine/charts/traefik/crds/ingressroute-udp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4efa15da88878fd3ec4ee7e2e91f6400fe9479e1 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/crds/ingressroute-udp.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressrouteudps.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRouteUDP + plural: ingressrouteudps + singular: ingressrouteudp + scope: Namespaced diff --git a/helm-charts/submarine/charts/traefik/crds/ingressroute.yaml b/helm-charts/submarine/charts/traefik/crds/ingressroute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d42e4576aa70ab0a08130b03c733c14881f590bc --- /dev/null +++ b/helm-charts/submarine/charts/traefik/crds/ingressroute.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ingressroutes.traefik.containo.us +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: IngressRoute + plural: ingressroutes + singular: ingressroute + scope: Namespaced diff --git a/helm-charts/submarine/charts/traefik/crds/middlewares.yaml b/helm-charts/submarine/charts/traefik/crds/middlewares.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b19370d30813606a7fd87352aa96c485f79ae48 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/crds/middlewares.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: middlewares.traefik.containo.us +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: Middleware + plural: middlewares + singular: middleware + scope: Namespaced diff --git a/helm-charts/submarine/charts/traefik/crds/tls-options.yaml b/helm-charts/submarine/charts/traefik/crds/tls-options.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bc26976fd818b7bb402c0b91be31e5e97e5c826 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/crds/tls-options.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tlsoptions.traefik.containo.us +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TLSOption + plural: tlsoptions + singular: tlsoption + scope: Namespaced diff --git a/helm-charts/submarine/charts/traefik/crds/tls-stores.yaml b/helm-charts/submarine/charts/traefik/crds/tls-stores.yaml new file mode 100644 index 0000000000000000000000000000000000000000..728345460c57e92bfcf1c059bdfeb0737315948a --- /dev/null +++ b/helm-charts/submarine/charts/traefik/crds/tls-stores.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tlsstores.traefik.containo.us + +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TLSStore + plural: tlsstores + singular: tlsstore + scope: Namespaced diff --git a/helm-charts/submarine/charts/traefik/crds/traefik-services.yaml b/helm-charts/submarine/charts/traefik/crds/traefik-services.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7eb8ab9a99abcfd4d09ee59777e7dae93433c1f8 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/crds/traefik-services.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: traefikservices.traefik.containo.us +spec: + group: traefik.containo.us + version: v1alpha1 + names: + kind: TraefikService + plural: traefikservices + singular: traefikservice + scope: Namespaced diff --git a/helm-charts/submarine/charts/traefik/templates/_helpers.tpl b/helm-charts/submarine/charts/traefik/templates/_helpers.tpl new file mode 100644 index 0000000000000000000000000000000000000000..d599823ea3180a6902c57666fe463201173e46c9 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{/* +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "traefik.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "traefik.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "traefik.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +The name of the service account to use +*/}} +{{- define "traefik.serviceAccountName" -}} +{{- default (include "traefik.fullname" .) .Values.serviceAccount.name -}} +{{- end -}} + +{{/* +Construct the path for the providers.kubernetesingress.ingressendpoint.publishedservice. +By convention this will simply use the / to match the name of the +service generated. +Users can provide an override for an explicit service they want bound via `.Values.providers.kubernetesIngress.publishedService.pathOverride` +*/}} +{{- define "providers.kubernetesIngress.publishedServicePath" -}} +{{- $defServiceName := printf "%s/%s" .Release.Namespace (include "traefik.fullname" .) -}} +{{- $servicePath := default $defServiceName .Values.providers.kubernetesIngress.publishedService.pathOverride }} +{{- print $servicePath | trimSuffix "-" -}} +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/dashboard-hook-ingressroute.yaml b/helm-charts/submarine/charts/traefik/templates/dashboard-hook-ingressroute.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9565b93764aa49ff29b330d91304c6877be2127d --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/dashboard-hook-ingressroute.yaml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.ingressRoute.dashboard.enabled -}} +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: {{ template "traefik.fullname" . }}-dashboard + annotations: + helm.sh/hook: "post-install,post-upgrade" + {{- with .Values.ingressRoute.dashboard.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.ingressRoute.dashboard.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + entryPoints: + - traefik + routes: + - match: PathPrefix(`/dashboard`) || PathPrefix(`/api`) + kind: Rule + services: + - name: api@internal + kind: TraefikService +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/deployment.yaml b/helm-charts/submarine/charts/traefik/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6e65d0f03b9f2e002e1ce3cdc008d691126020 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/deployment.yaml @@ -0,0 +1,260 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.deployment.enabled -}} + {{- if gt (int .Values.deployment.replicas) 1 -}} + {{- with .Values.additionalArguments -}} + {{- range . -}} + {{- if contains ".acme." . -}} + {{- fail (printf "You can not enabled acme if you set more than one traefik replica") -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + {{- with .Values.deployment.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ default 1 .Values.deployment.replicas }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + strategy: + type: RollingUpdate + rollingUpdate: + {{- with .Values.rollingUpdate }} + {{- toYaml . | nindent 6 }} + {{- end }} + template: + metadata: + annotations: + {{- with .Values.deployment.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + serviceAccountName: {{ include "traefik.serviceAccountName" . }} + terminationGracePeriodSeconds: 60 + hostNetwork: {{ .Values.hostNetwork }} + {{- with .Values.deployment.dnsPolicy }} + dnsPolicy: {{ . }} + {{- end }} + {{- with .Values.deployment.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + - image: {{ .Values.image.name }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: {{ template "traefik.fullname" . }} + resources: + {{- with .Values.resources }} + {{- toYaml . | nindent 10 }} + {{- end }} + readinessProbe: + httpGet: + path: /ping + port: {{ .Values.ports.traefik.port }} + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + httpGet: + path: /ping + port: {{ .Values.ports.traefik.port }} + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + ports: + {{- range $name, $config := .Values.ports }} + {{- if $config }} + - name: {{ $name | quote }} + containerPort: {{ $config.port }} + {{- if $config.hostPort }} + hostPort: {{ $config.hostPort }} + {{- end }} + {{- if $config.hostIP }} + hostIP: {{ $config.hostIP }} + {{- end }} + protocol: {{ default "TCP" $config.protocol | quote }} + {{- end }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + - name: data + mountPath: {{ .Values.persistence.path }} + {{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + - name: tmp + mountPath: /tmp + {{- range .Values.volumes }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: true + {{- end }} + args: + {{- with .Values.globalArguments }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- range $name, $config := .Values.ports }} + {{- if $config }} + - "--entryPoints.{{$name}}.address=:{{ $config.port }}/{{ default "tcp" $config.protocol | lower }}" + {{- end }} + {{- end }} + - "--api.dashboard=true" + - "--ping=true" + {{- if .Values.providers.kubernetesCRD.enabled }} + - "--providers.kubernetescrd" + {{- end }} + {{- if .Values.providers.kubernetesIngress.enabled }} + - "--providers.kubernetesingress" + {{- if and .Values.service.enabled .Values.providers.kubernetesIngress.publishedService.enabled }} + - "--providers.kubernetesingress.ingressendpoint.publishedservice={{ template "providers.kubernetesIngress.publishedServicePath" . }}" + {{- end }} + {{- end }} + {{- if and .Values.rbac.enabled .Values.rbac.namespaced }} + - "--providers.kubernetescrd.namespaces={{ .Release.Namespace }}" + - "--providers.kubernetesingress.namespaces={{ .Release.Namespace }}" + {{- end }} + {{- range $entrypoint, $config := $.Values.ports }} + {{- if $config.redirectTo }} + {{- $toPort := index $.Values.ports $config.redirectTo }} + - "--entrypoints.{{ $entrypoint }}.http.redirections.entryPoint.to=:{{ $toPort.exposedPort }}" + - "--entrypoints.{{ $entrypoint }}.http.redirections.entryPoint.scheme=https" + {{- end }} + {{- end }} + {{- with .Values.logs }} + {{- if .general.format }} + - "--log.format={{ .general.format }}" + {{- end }} + {{- if ne .general.level "ERROR" }} + - "--log.level={{ .general.level | upper }}" + {{- end }} + {{- if .access.enabled }} + - "--accesslog=true" + {{- if .access.format }} + - "--accesslog.format={{ .access.format }}" + {{- end }} + {{- if .access.bufferingsize }} + - "--accesslog.bufferingsize={{ .access.bufferingsize }}" + {{- end }} + {{- if .access.filters }} + {{- if .access.filters.statuscodes }} + - "--accesslog.filters.statuscodes={{ .access.filters.statuscodes }}" + {{- end }} + {{- if .access.filters.retryattempts }} + - "--accesslog.filters.retryattempts" + {{- end }} + {{- if .access.filters.minduration }} + - "--accesslog.filters.minduration={{ .access.filters.minduration }}" + {{- end }} + {{- end }} + - "--accesslog.fields.defaultmode={{ .access.fields.general.defaultmode }}" + {{- range $fieldname, $fieldaction := .access.fields.general.names }} + - "--accesslog.fields.names.{{ $fieldname }}={{ $fieldaction }}" + {{- end }} + - "--accesslog.fields.headers.defaultmode={{ .access.fields.headers.defaultmode }}" + {{- range $fieldname, $fieldaction := .access.fields.headers.names }} + - "--accesslog.fields.headers.names.{{ $fieldname }}={{ $fieldaction }}" + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.additionalArguments }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.envFrom }} + envFrom: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.deployment.additionalContainers }} + {{- toYaml .Values.deployment.additionalContainers | nindent 6 }} + {{- end }} + volumes: + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ default (include "traefik.fullname" .) .Values.persistence.existingClaim }} + {{- else }} + emptyDir: {} + {{- end }} + - name: tmp + emptyDir: {} + {{- range .Values.volumes }} + - name: {{ .name }} + {{- if eq .type "secret" }} + secret: + secretName: {{ .name }} + {{- else if eq .type "configMap" }} + configMap: + name: {{ .name }} + {{- end }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/hpa.yaml b/helm-charts/submarine/charts/traefik/templates/hpa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ecdd16f2e9f663298e3635f0591664f8ebb7aa2 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/hpa.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "traefik.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: +{{ toYaml .Values.autoscaling.metrics | indent 4 }} +{{- end }} diff --git a/helm-charts/submarine/charts/traefik/templates/pod-disruption-budget.yaml b/helm-charts/submarine/charts/traefik/templates/pod-disruption-budget.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa460e91a7715c9895bec6f90b77a69fe70aadfd --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/pod-disruption-budget.yaml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.podDisruptionBudget.enabled -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable | int }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable | int }} + {{- end }} +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/pvc.yaml b/helm-charts/submarine/charts/traefik/templates/pvc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0993ea10194d175597e8de7180d6385bdbe4e53f --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/pvc.yaml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "traefik.fullname" . }} + annotations: + {{- with .Values.persistence.annotations }} + {{ toYaml . | indent 4 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} +{{- end -}} + diff --git a/helm-charts/submarine/charts/traefik/templates/rbac/cluster-role-binding.yaml b/helm-charts/submarine/charts/traefik/templates/rbac/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1c5c958b5f4f8b4bfd761ffa653e77c322878a1 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/rbac/cluster-role-binding.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.rbac.enabled (not .Values.rbac.namespaced) }} +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "traefik.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "traefik.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/rbac/cluster-role.yaml b/helm-charts/submarine/charts/traefik/templates/rbac/cluster-role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..191c9ec883c84a9df41b0715e9cdf1b8d7932e97 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/rbac/cluster-role.yaml @@ -0,0 +1,66 @@ +{{- if and .Values.rbac.enabled (not .Values.rbac.namespaced) -}} +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: + - apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - traefik.containo.us + resources: + - ingressroutes + - ingressroutetcps + - ingressrouteudps + - middlewares + - tlsoptions + - tlsstores + - traefikservices + verbs: + - get + - list + - watch +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/rbac/role-binding.yaml b/helm-charts/submarine/charts/traefik/templates/rbac/role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3d6996a739c16895eb3bc0c9b8768268f6db8b7 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/rbac/role-binding.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.rbac.enabled .Values.rbac.namespaced }} +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "traefik.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "traefik.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/rbac/role.yaml b/helm-charts/submarine/charts/traefik/templates/rbac/role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0775f4092898b98efe351eab087bbff4b727c99 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/rbac/role.yaml @@ -0,0 +1,66 @@ +{{- if and .Values.rbac.enabled .Values.rbac.namespaced }} +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: + - apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - traefik.containo.us + resources: + - ingressroutes + - ingressroutetcps + - ingressrouteudps + - middlewares + - tlsoptions + - tlsstores + - traefikservices + verbs: + - get + - list + - watch +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/rbac/service-account.yaml b/helm-charts/submarine/charts/traefik/templates/rbac/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1df7115927ed23ae120b5331fb6db978c5f12e2e --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/rbac/service-account.yaml @@ -0,0 +1,31 @@ +{{- if not .Values.serviceAccount.name -}} +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +kind: ServiceAccount +apiVersion: v1 +metadata: + name: {{ include "traefik.serviceAccountName" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + {{- with .Values.serviceAccountAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/templates/service.yaml b/helm-charts/submarine/charts/traefik/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40972e457a4a7d59ee6ea43768476a2072a97ded --- /dev/null +++ b/helm-charts/submarine/charts/traefik/templates/service.yaml @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.service.enabled -}} + +{{ $tcpPorts := dict }} +{{ $udpPorts := dict }} +{{- range $name, $config := .Values.ports }} + {{- if eq (toString $config.protocol) "UDP" }} + {{ $_ := set $udpPorts $name $config }} + {{- else }} + {{ $_ := set $tcpPorts $name $config }} + {{- end }} +{{- end }} + +apiVersion: v1 +kind: List +items: +{{- if $tcpPorts }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ template "traefik.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- $type := default "LoadBalancer" .Values.service.type }} + type: {{ $type }} + {{- with .Values.service.spec }} + {{- toYaml . | nindent 6 }} + {{- end }} + selector: + app.kubernetes.io/name: {{ template "traefik.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + {{- range $name, $config := $tcpPorts }} + {{- if $config.expose }} + - port: {{ default $config.port $config.exposedPort }} + name: {{ $name }} + targetPort: {{ $name | quote }} + protocol: {{ default "TCP" $config.protocol | quote }} + {{- if $config.nodePort }} + nodePort: {{ $config.nodePort }} + {{- end }} + {{- end }} + {{- end }} + {{- if eq $type "LoadBalancer" }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 6 }} + {{- end -}} + {{- end -}} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 6 }} + {{- end -}} +{{- end }} + +{{- if $udpPorts }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ template "traefik.fullname" . }}-udp + labels: + app.kubernetes.io/name: {{ template "traefik.name" . }} + helm.sh/chart: {{ template "traefik.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- $type := default "LoadBalancer" .Values.service.type }} + type: {{ $type }} + {{- with .Values.service.spec }} + {{- toYaml . | nindent 6 }} + {{- end }} + selector: + app.kubernetes.io/name: {{ template "traefik.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + {{- range $name, $config := $udpPorts }} + {{- if $config.expose }} + - port: {{ default $config.port $config.exposedPort }} + name: {{ $name }} + targetPort: {{ $name | quote }} + protocol: {{ default "UDP" $config.protocol | quote }} + {{- if $config.nodePort }} + nodePort: {{ $config.nodePort }} + {{- end }} + {{- end }} + {{- end }} + {{- if eq $type "LoadBalancer" }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 6 }} + {{- end -}} + {{- end -}} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 6 }} + {{- end -}} +{{- end }} +{{- end -}} diff --git a/helm-charts/submarine/charts/traefik/values.yaml b/helm-charts/submarine/charts/traefik/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17d164b169bc6b3b4fec8ad2f5d65969c6c0bac3 --- /dev/null +++ b/helm-charts/submarine/charts/traefik/values.yaml @@ -0,0 +1,336 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default values for Traefik +image: + name: traefik + tag: 2.2.8 + pullPolicy: IfNotPresent + +# +# Configure the deployment +# +deployment: + enabled: true + # Number of pods of the deployment + replicas: 1 + # Additional deployment annotations (e.g. for jaeger-operator sidecar injection) + annotations: {} + # Additional pod annotations (e.g. for mesh injection or prometheus scraping) + podAnnotations: {} + # Additional containers (e.g. for metric offloading sidecars) + additionalContainers: [] + # Additional initContainers (e.g. for setting file permission as shown below) + initContainers: [] + # The "volume-permissions" init container is required if you run into permission issues. + # Related issue: https://github.com/containous/traefik/issues/6972 + # - name: volume-permissions + # image: busybox:1.31.1 + # command: ["sh", "-c", "chmod -Rv 600 /data/*"] + # volumeMounts: + # - name: data + # mountPath: /data + # Custom pod DNS policy. Apply if `hostNetwork: true` + # dnsPolicy: ClusterFirstWithHostNet + +# Pod disruption budget +podDisruptionBudget: + enabled: false + # maxUnavailable: 1 + # minAvailable: 0 + +# Create an IngressRoute for the dashboard +ingressRoute: + dashboard: + enabled: false + # Additional ingressRoute annotations (e.g. for kubernetes.io/ingress.class) + annotations: {} + # Additional ingressRoute labels (e.g. for filtering IngressRoute by custom labels) + labels: {} + +rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 + + +# +# Configure providers +# +providers: + kubernetesCRD: + enabled: true + kubernetesIngress: + enabled: true + # IP used for Kubernetes Ingress endpoints + publishedService: + enabled: false + # Published Kubernetes Service to copy status from. Format: namespace/servicename + # By default this Traefik service + # pathOverride: "" + +# +# Add volumes to the traefik pod. +# This can be used to mount a cert pair or a configmap that holds a config.toml file. +# After the volume has been mounted, add the configs into traefik by using the `additionalArguments` list below, eg: +# additionalArguments: +# - "--providers.file.filename=/config/dynamic.toml" +volumes: [] +# - name: public-cert +# mountPath: "/certs" +# type: secret +# - name: configs +# mountPath: "/config" +# type: configMap + +# Logs +# https://docs.traefik.io/observability/logs/ +logs: + # Traefik logs concern everything that happens to Traefik itself (startup, configuration, events, shutdown, and so on). + general: + # By default, the logs use a text format (common), but you can + # also ask for the json format in the format option + # format: json + # By default, the level is set to ERROR. Alternative logging levels are DEBUG, PANIC, FATAL, ERROR, WARN, and INFO. + level: ERROR + access: + # To enable access logs + enabled: false + # By default, logs are written using the Common Log Format (CLF). + # To write logs in JSON, use json in the format option. + # If the given format is unsupported, the default (CLF) is used instead. + # format: json + # To write the logs in an asynchronous fashion, specify a bufferingSize option. + # This option represents the number of log lines Traefik will keep in memory before writing + # them to the selected output. In some cases, this option can greatly help performances. + # bufferingSize: 100 + # Filtering https://docs.traefik.io/observability/access-logs/#filtering + filters: {} + # statuscodes: "200,300-302" + # retryattempts: true + # minduration: 10ms + # Fields + # https://docs.traefik.io/observability/access-logs/#limiting-the-fieldsincluding-headers + fields: + general: + defaultmode: keep + names: {} + # Examples: + # ClientUsername: drop + headers: + defaultmode: drop + names: {} + # Examples: + # User-Agent: redact + # Authorization: drop + # Content-Type: keep + +globalArguments: + - "--global.checknewversion" + - "--global.sendanonymoususage" + +# +# Configure Traefik static configuration +# Additional arguments to be passed at Traefik's binary +# All available options available on https://docs.traefik.io/reference/static-configuration/cli/ +## Use curly braces to pass values: `helm install --set="additionalArguments={--providers.kubernetesingress.ingressclass=traefik-internal,--log.level=DEBUG}"` +additionalArguments: [] +# - "--providers.kubernetesingress.ingressclass=traefik-internal" +# - "--log.level=DEBUG" + +# Environment variables to be passed to Traefik's binary +env: [] +# - name: SOME_VAR +# value: some-var-value +# - name: SOME_VAR_FROM_CONFIG_MAP +# valueFrom: +# configMapRef: +# name: configmap-name +# key: config-key +# - name: SOME_SECRET +# valueFrom: +# secretKeyRef: +# name: secret-name +# key: secret-key + +envFrom: [] +# - configMapRef: +# name: config-map-name +# - secretRef: +# name: secret-name + +# Configure ports +ports: + # The name of this one can't be changed as it is used for the readiness and + # liveness probes, but you can adjust its config to your liking + traefik: + port: 9000 + # Use hostPort if set. + # hostPort: 9000 + # + # Use hostIP if set. If not set, Kubernetes will default to 0.0.0.0, which + # means it's listening on all your interfaces and all your IPs. You may want + # to set this value if you need traefik to listen on specific interface + # only. + # hostIP: 192.168.100.10 + + # Defines whether the port is exposed if service.type is LoadBalancer or + # NodePort. + # + # You SHOULD NOT expose the traefik port on production deployments. + # If you want to access it from outside of your cluster, + # use `kubectl proxy` or create a secure ingress + expose: false + # The exposed port for this service + exposedPort: 9000 + # The port protocol (TCP/UDP) + protocol: TCP + web: + port: 8000 + # hostPort: 8000 + expose: true + exposedPort: 80 + # The port protocol (TCP/UDP) + protocol: TCP + # Use nodeport if set. This is useful if you have configured Traefik in a + # LoadBalancer + nodePort: 32080 + # Port Redirections + # Added in 2.2, you can make permanent redirects via entrypoints. + # https://docs.traefik.io/routing/entrypoints/#redirection + # redirectTo: websecure + websecure: + port: 8443 + # hostPort: 8443 + expose: true + exposedPort: 443 + # The port protocol (TCP/UDP) + protocol: TCP + # nodePort: 32443 + +# Options for the main traefik service, where the entrypoints traffic comes +# from. +service: + enabled: true + type: NodePort + # Additional annotations (e.g. for cloud provider specific config) + annotations: {} + # Additional entries here will be added to the service spec. Cannot contains + # type, selector or ports entries. + spec: {} + # externalTrafficPolicy: Cluster + # loadBalancerIP: "1.2.3.4" + # clusterIP: "2.3.4.5" + loadBalancerSourceRanges: [] + # - 192.168.0.1/32 + # - 172.16.0.0/16 + externalIPs: [] + # - 1.2.3.4 + +## Create HorizontalPodAutoscaler object. +## +autoscaling: + enabled: false +# minReplicas: 1 +# maxReplicas: 10 +# metrics: +# - type: Resource +# resource: +# name: cpu +# targetAverageUtilization: 60 +# - type: Resource +# resource: +# name: memory +# targetAverageUtilization: 60 + +# Enable persistence using Persistent Volume Claims +# ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +# After the pvc has been mounted, add the configs into traefik by using the `additionalArguments` list below, eg: +# additionalArguments: +# - "--certificatesresolvers.le.acme.storage=/data/acme.json" +# It will persist TLS certificates. +persistence: + enabled: false +# existingClaim: "" + accessMode: ReadWriteOnce + size: 128Mi + # storageClass: "" + path: /data + annotations: {} + # subPath: "" # only mount a subpath of the Volume into the pod + +# If hostNetwork is true, runs traefik in the host network namespace +# To prevent unschedulabel pods due to port collisions, if hostNetwork=true +# and replicas>1, a pod anti-affinity is recommended and will be set if the +# affinity is left as default. +hostNetwork: false + +# Whether Role Based Access Control objects like roles and rolebindings should be created +rbac: + enabled: true + + # If set to false, installs ClusterRole and ClusterRoleBinding so Traefik can be used across namespaces. + # If set to true, installs namespace-specific Role and RoleBinding and requires provider configuration be set to that same namespace + namespaced: false + +# The service account the pods will use to interact with the Kubernetes API +serviceAccount: + # If set, an existing service account is used + # If not set, a service account is created automatically using the fullname template + name: "" + +# Additional serviceAccount annotations (e.g. for oidc authentication) +serviceAccountAnnotations: {} + +resources: {} + # requests: + # cpu: "100m" + # memory: "50Mi" + # limits: + # cpu: "300m" + # memory: "150Mi" +affinity: {} +# # This example pod anti-affinity forces the scheduler to put traefik pods +# # on nodes where no other traefik pods are scheduled. +# # It should be used when hostNetwork: true to prevent port conflicts +# podAntiAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# - labelSelector: +# matchExpressions: +# - key: app +# operator: In +# values: +# - {{ template "traefik.name" . }} +# topologyKey: failure-domain.beta.kubernetes.io/zone +nodeSelector: {} +tolerations: [] + +# Pods can have priority. +# Priority indicates the importance of a Pod relative to other Pods. +priorityClassName: "" + +# Set the container security context +# To run the container with ports below 1024 this will need to be adjust to run as root +securityContext: + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + +podSecurityContext: + fsGroup: 65532 diff --git a/helm-charts/submarine/templates/rbac.yaml b/helm-charts/submarine/templates/rbac.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acad2e170c040e1836643274d8e02a0f6f5e7190 --- /dev/null +++ b/helm-charts/submarine/templates/rbac.yaml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "{{ .Values.submarine.server.name }}" +rules: +- apiGroups: + - kubeflow.org + resources: + - tfjobs + - tfjobs/status + - pytorchjobs + - pytorchjobs/status + - notebooks + - notebooks/status + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - traefik.containo.us + resources: + - ingressroutes + verbs: + - get + - list + - watch + - create + - delete + - deletecollection + - patch + - update +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - '*' +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: "{{ .Values.submarine.server.name }}" +subjects: +- kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: "{{ .Values.submarine.server.name }}" +roleRef: + kind: ClusterRole + name: "{{ .Values.submarine.server.name }}" + apiGroup: "" diff --git a/helm-charts/submarine/templates/submarine-database.yaml b/helm-charts/submarine/templates/submarine-database.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02803a87aac16f773b7a7122e392f9a622013e5d --- /dev/null +++ b/helm-charts/submarine/templates/submarine-database.yaml @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "{{ .Values.submarine.database.name }}" +spec: + replicas: {{ .Values.submarine.database.replicas }} + selector: + matchLabels: + app: "{{ .Values.submarine.database.name }}" + template: + metadata: + labels: + app: "{{ .Values.submarine.database.name }}" + spec: + containers: + - name: "{{ .Values.submarine.database.name }}" + image: "{{ .Values.submarine.database.image }}" + imagePullPolicy: {{ .Values.submarine.database.imagePullPolicy }} + ports: + - containerPort: 3306 + env: + - name: MYSQL_ROOT_PASSWORD + value: "{{ .Values.submarine.database.mysqlRootPassword }}" + +--- +apiVersion: v1 +kind: Service +metadata: + name: "{{ .Values.submarine.database.name }}" +spec: + ports: + - name: "{{ .Values.submarine.database.name }}" + port: 3306 + targetPort: {{ .Values.submarine.database.servicePort }} + selector: + app: "{{ .Values.submarine.database.name }}" diff --git a/helm-charts/submarine/templates/submarine-ingress.yaml b/helm-charts/submarine/templates/submarine-ingress.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bddae3899e28b30e72396a9db0390b4f68c1588 --- /dev/null +++ b/helm-charts/submarine/templates/submarine-ingress.yaml @@ -0,0 +1,16 @@ +{{ if .Values.submarine.traefik.enabled }} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ .Values.submarine.server.name}}-ingress + namespace: {{ .Release.namespace }} + +spec: + rules: + - http: + paths: + - path: / + backend: + serviceName: {{ .Values.submarine.server.name }} + servicePort: {{ .Values.submarine.server.servicePort }} +{{ end }} diff --git a/helm-charts/submarine/templates/submarine-server.yaml b/helm-charts/submarine/templates/submarine-server.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03dada2ab8021dd1fa76b1ca10c65d5b0fa15545 --- /dev/null +++ b/helm-charts/submarine/templates/submarine-server.yaml @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: "{{ .Values.submarine.server.name }}" + labels: + run: "{{ .Values.submarine.server.name }}" +spec: + ports: + - port: 8080 + targetPort: {{ .Values.submarine.server.servicePort }} + protocol: TCP + selector: + run: "{{ .Values.submarine.server.name }}" + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Values.submarine.server.name }}" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "{{ .Values.submarine.server.name }}" +spec: + selector: + matchLabels: + run: "{{ .Values.submarine.server.name }}" + replicas: {{ .Values.submarine.server.replicas }} + template: + metadata: + labels: + run: "{{ .Values.submarine.server.name }}" + spec: + serviceAccountName: "{{ .Values.submarine.server.name }}" + containers: + - name: "{{ .Values.submarine.server.name }}" + env: + - name: SUBMARINE_SERVER_PORT + value: "8080" + - name: SUBMARINE_SERVER_PORT_8080_TCP + value: "8080" + - name: SUBMARINE_SERVER_DNS_NAME + value: "{{ .Values.submarine.server.name }}.{{ .Release.Namespace }}" + - name: K8S_APISERVER_URL + value: "kubernetes.default.svc" + + image: "{{ .Values.submarine.server.image }}" + imagePullPolicy: {{ .Values.submarine.server.imagePullPolicy }} + command: ["/bin/bash", "-c"] + args: ["/opt/submarine-current/bin/submarine.sh start getMysqlJar"] + ports: + - containerPort: 8080 + diff --git a/helm-charts/submarine/values.yaml b/helm-charts/submarine/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd64b90ec9f1fcca62666c833e44d7ec2baef77e --- /dev/null +++ b/helm-charts/submarine/values.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +submarine: + server: + imagePullPolicy: IfNotPresent + replicas: 1 + name: submarine-server + image: apache/submarine:server-0.5.0 + servicePort: 8080 + database: + imagePullPolicy: IfNotPresent + replicas: 1 + name: submarine-database + image: apache/submarine:database-0.5.0 + servicePort: 3306 + mysqlRootPassword: password + traefik: + enabled: true diff --git a/licenses-binary/LICENSE-asm.txt b/licenses-binary/LICENSE-asm.txt new file mode 100644 index 0000000000000000000000000000000000000000..631ee53c53d425652dfa91ef7b669bf9bdf2e092 --- /dev/null +++ b/licenses-binary/LICENSE-asm.txt @@ -0,0 +1,27 @@ +ASM: a very small and fast Java bytecode manipulation framework +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/licenses-binary/LICENSE-hamcrest.txt b/licenses-binary/LICENSE-hamcrest.txt new file mode 100644 index 0000000000000000000000000000000000000000..e3d4feb9cbba9303582fbdb299f6a20a2de2f7fb --- /dev/null +++ b/licenses-binary/LICENSE-hamcrest.txt @@ -0,0 +1,27 @@ +BSD License + +Copyright (c) 2000-2006, www.hamcrest.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. Redistributions in binary form must reproduce +the above copyright notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the distribution. + +Neither the name of Hamcrest nor the names of its contributors may be used to endorse +or promote products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. \ No newline at end of file diff --git a/licenses-binary/LICENSE-jgit.txt b/licenses-binary/LICENSE-jgit.txt new file mode 100644 index 0000000000000000000000000000000000000000..8a4e20e33d238f3824413dc00dfce928efbfc89e --- /dev/null +++ b/licenses-binary/LICENSE-jgit.txt @@ -0,0 +1,37 @@ +This program and the accompanying materials are made available +under the terms of the Eclipse Distribution License v1.0 which +accompanies this distribution, is reproduced below, and is +available at http://www.eclipse.org/org/documents/edl-v10.php + +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + +- Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +- Neither the name of the Eclipse Foundation, Inc. nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/licenses-binary/LICENSE-pagehelper.txt b/licenses-binary/LICENSE-pagehelper.txt new file mode 100644 index 0000000000000000000000000000000000000000..749be647a0b9606622c0517b620efe3a4b53ea68 --- /dev/null +++ b/licenses-binary/LICENSE-pagehelper.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017 abel533@gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE-paranamer.txt b/licenses-binary/LICENSE-paranamer.txt new file mode 100644 index 0000000000000000000000000000000000000000..991c5644c147d2344155594235b855d49d23c806 --- /dev/null +++ b/licenses-binary/LICENSE-paranamer.txt @@ -0,0 +1,29 @@ +[ ParaNamer used to be 'Pubic Domain', but since it includes a small piece of ASM it is now the same license as that: BSD ] + + Portions copyright (c) 2006-2018 Paul Hammant & ThoughtWorks Inc + Portions copyright (c) 2000-2007 INRIA, France Telecom + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/licenses-binary/LICENSE-protobuf.txt b/licenses-binary/LICENSE-protobuf.txt new file mode 100644 index 0000000000000000000000000000000000000000..97a6e3d199a5fbefced7ef657a59be8bd8a76f70 --- /dev/null +++ b/licenses-binary/LICENSE-protobuf.txt @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. \ No newline at end of file diff --git a/licenses-binary/LICENSE-slf4j.txt b/licenses-binary/LICENSE-slf4j.txt new file mode 100644 index 0000000000000000000000000000000000000000..ba8ac4d43938f09c4220834bd47ce764139cfd1f --- /dev/null +++ b/licenses-binary/LICENSE-slf4j.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2017 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE-tony.txt b/licenses-binary/LICENSE-tony.txt new file mode 100644 index 0000000000000000000000000000000000000000..181ce870e4adf663d77eb99c617fb2f89d89c006 --- /dev/null +++ b/licenses-binary/LICENSE-tony.txt @@ -0,0 +1,28 @@ +BSD 2-CLAUSE LICENSE + +Copyright 2018 LinkedIn Corporation. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..4a8a17c6e20de979393e221362d2a8d407df28cb --- /dev/null +++ b/pom.xml @@ -0,0 +1,586 @@ + + + + + 4.0.0 + + org.apache.submarine + submarine + 0.5.0 + pom + Submarine + Submarine project + https://submarine.apache.org + + + org.apache + apache + 17 + + + + + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + aliyun + Apache Development Snapshot Repository + http://maven.aliyun.com/nexus/content/groups/public/ + + true + + + + + + + 1.8 + 1.11.8 + + + 2.4 + 3.1 + 3.2.1 + 1.3.1 + 2.17 + 2.3 + 1.8 + 3.0.2 + 3.3.0 + + + org.apache.submarine.shaded + + + 0.3.25 + 2.27 + 1.13 + 9.4.18.v20190429 + 2.1 + 5.1.10 + + 2.8.5 + 2.11.0 + 2.11.0 + 2.11.0 + 1.10 + 3.1 + + 3.2.2 + 3.2.8 + 5.1.39 + 1.25.0 + + + 1.6 + v10.12.0 + v1.10.1 + 6.4.1 + 2.1.2 + + ${basedir}/../hadoop-common-project/hadoop-common/target + 1.7.25 + 1.2.17 + 1.1.3 + 1.2 + 1.16 + 4.4.4 + 4.5.2 + 2.6 + 3.4 + 2.5 + 1.5 + 4.12 + 3.8.1 + 0.7.1 + 1.3.9 + 2.23.4 + 1.6.4 + 22.0 + 6.4 + 1.8.2 + 4.5.2 + 2.9.10 + 2.6 + 1.1.0 + 1.9.13 + 1.4 + 1.3.2 + 3.2.2 + 7.9 + 1.3.7 + 10.15.1.3 + 0.9.0-preview1 + 5.5.1.201910021850-r + 3.1.5 + 2.11.8 + 2.11 + 2.3.6 + + 6.0.1 + 2.27 + + 2.17 + + 0.7 + + + + submarine-commons + submarine-client + submarine-cloud + submarine-server + submarine-all + submarine-workbench + submarine-dist + submarine-test + submarine-security/spark-security + + + + + + log4j + log4j + ${log4j.version} + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + junit + junit + ${junit.version} + + + org.mockito + mockito-core + ${mockito.version} + + + org.testng + testng + ${testng.version} + + + + + io.kubernetes + client-java + ${k8s.client-java.version} + + + + + org.apache.hadoop + hadoop-yarn-services-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-yarn-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + com.google.guava + guava + ${guava.version} + + + javax.xml.bind + jaxb-api + ${jaxb-api.version} + + + org.apache.commons + commons-compress + ${commons-compress.version} + + + com.google.inject.extensions + guice-servlet + ${guice-servlet.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson-annotations.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson-annotations.version} + + + com.google.inject + guice + ${guice.version} + + + com.nimbusds + nimbus-jose-jwt + ${nimbus-jose-jwt.version} + + + commons-io + commons-io + ${commons-io.version} + + + commons-configuration + commons-configuration + ${commons-configuration.version} + + + commons-lang + commons-lang + ${commons-lang.version} + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + commons-httpclient + commons-httpclient + ${commons-httpclient.version} + + + javax.ws.rs + javax.ws.rs-api + ${javax.ws.rsapi.version} + + + org.codehaus.jackson + jackson-xc + ${codehaus-jackson.version} + + + org.codehaus.jackson + jackson-jaxrs + ${codehaus-jackson.version} + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + com.google.code.gson + gson + ${gson.version} + + + commons-logging + commons-logging + ${commons.logging.version} + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + + + + + hadoop-3.2 + + 3.2.1 + 2.2.11 + 1.19 + 4.0 + 4.0 + 3.4.13 + 27.0-jre + 3.0.2 + hadoop-3.2 + + + submarine-server/server-submitter/submitter-yarnservice + + + + + hadoop-3.1 + + 3.1.3 + 2.2.11 + 1.19 + 4.0 + 4.0 + 3.4.13 + hadoop-3.1 + + + submarine-server/server-submitter/submitter-yarnservice + + + + + hadoop-2.10 + + false + + + 2.10.0 + 2.2.2 + 1.19 + 3.0 + 3.0 + 3.4.9 + hadoop-2.10 + + + + + hadoop-2.9 + + true + + + 2.9.2 + 2.2.11 + 1.4.1 + 3.0 + 3.0 + 3.4.6 + hadoop-2.9 + + + + + clover + + false + + clover + + + + + com.cenqua.clover + clover + + + + + + rat + + + skipRat + + + + + + + + + maven-compiler-plugin + ${plugin.compiler.version} + + ${java.version} + ${java.version} + + + + + maven-enforcer-plugin + ${plugin.enforcer.version} + + + enforce + + + + + true + + + enforce + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${plugin.checkstyle.version} + + true + false + true + ${project.build.sourceDirectory} + ${project.build.testSourceDirectory} + dev-support/maven-config/checkstyle.xml + ${project.build.directory}/checkstyle-output.xml + ${project.build.sourceEncoding} + ${project.reporting.outputEncoding} + + + + checkstyle-fail-build + validate + + check + + + true + + + + checkstyle-gen-html-report + install + + checkstyle-aggregate + + + + + + + org.scalastyle + scalastyle-maven-plugin + 1.0.0 + + false + true + true + false + ${project.basedir}/src/test/scala + ${project.basedir}/src/main/scala + dev-support/maven-config/scalastyle-config.xml + ${project.build.directory}/scalastyle-output.xml + UTF-8 + + + + + check + + + + + + + org.apache.rat + apache-rat-plugin + true + + + .git/ + .github/** + .gitignore + .gitmodules + .editorconfig + LICENSE + license_header + **/*.iml + **/*.la + **/*.svg + **/*.png + **/*.avsc + **/log4j.properties + **/*.md + **/*.proto + **/*.txt + **/*.xml + **/*.html + **/*.xsd + **/*.json + **/*.conf + **/*.libsvm + **/*.yaml + **/*.yapf + **/*.libsvm + **/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo + **/src/test/resources/typicalHistFolder/job1/application123-1-1-user1-SUCCEEDED.jhist + **/conf/routes + **/submitter-yarnservice/target/** + **/src/test/resources/config + licenses/** + licenses-binary/** + NOTICE-binary + **/go.mod + **/go.sum + **/*.ipynb + + + + + + diff --git a/submarine-all/pom.xml b/submarine-all/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..9bf988d4c73c3b4fcb8e1fc3b03713ebe5d2a7eb --- /dev/null +++ b/submarine-all/pom.xml @@ -0,0 +1,258 @@ + + + + 4.0.0 + + org.apache.submarine + submarine + 0.5.0 + .. + + submarine-all + 0.5.0 + Submarine: All + + + + + org.apache.hadoop + hadoop-yarn-services-core + ${hadoop.version} + + + org.apache.avro + avro + ${avro.version} + + + + + + + org.apache.submarine + submarine-commons-runtime + ${project.version} + + + org.apache.submarine + submarine-client + ${project.version} + + + com.google.protobuf + protobuf-java + + + com.google.code.gson + gson + + + + + org.apache.submarine + submarine-submitter-yarn + ${project.version} + + + + + + hadoop-3.2 + + + org.apache.submarine + submarine-submitter-yarnservice + ${project.version} + + + org.apache.avro + avro + + + commons-codec + commons-codec + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + io.netty + netty + + + com.fasterxml.jackson.core + jackson-databind + + + + + + + + hadoop-3.1 + + + org.apache.submarine + submarine-submitter-yarnservice + ${project.version} + + + org.apache.avro + avro + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + + + hadoop-2.10 + + false + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + + + hadoop-2.9 + + true + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${plugin.shade.version} + + + package + + shade + + + + target/${project.artifactId}-${project.version}-${project.activeProfiles[0].id}.jar + + + classworlds:classworlds + junit:junit + jmock:* + *:xml-apis + org.apache.maven:lib:tests + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.apache.submarine.client.cli.Cli + + + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + + + + maven-enforcer-plugin + + + + + diff --git a/submarine-client/pom.xml b/submarine-client/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..ddc0fd82205bf259ce7c9a68309ed5b1202b93d3 --- /dev/null +++ b/submarine-client/pom.xml @@ -0,0 +1,155 @@ + + + + + + submarine + org.apache.submarine + 0.5.0 + + 4.0.0 + + submarine-client + 0.5.0 + Submarine: Client + + + + org.apache.submarine + submarine-commons-runtime + ${project.version} + + + com.google.protobuf + protobuf-java + + + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.apache.submarine + submarine-commons-runtime + ${project.version} + test-jar + test + + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.apache.commons + commons-lang3 + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + commons-lang + commons-lang + + + commons-collections + commons-collections + + + commons-logging + commons-logging + + + com.google.guava + guava + + + commons-codec + commons-codec + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + com.google.protobuf + protobuf-java + + + + + org.apache.submarine + submarine-commons-rpc + ${project.version} + + + com.google.code.findbugs + jsr305 + + + com.google.errorprone + error_prone_annotations + + + org.codehaus.mojo + animal-sniffer-annotations + + + + + + diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java new file mode 100644 index 0000000000000000000000000000000000000000..177bf160abe447beebafa51214bc0357a74d3e34 --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli; + +import org.apache.commons.cli.ParseException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.submarine.commons.runtime.ClientContext; +import org.apache.submarine.commons.runtime.exception.SubmarineException; + +import java.io.IOException; + +public abstract class AbstractCli implements Tool { + protected ClientContext clientContext; + + public AbstractCli(ClientContext cliContext) { + this.clientContext = cliContext; + } + + @Override + public abstract int run(String[] args) + throws ParseException, IOException, YarnException, InterruptedException, + SubmarineException; + + @Override + public void setConf(Configuration conf) { + clientContext.setYarnConfig(conf); + } + + @Override + public Configuration getConf() { + return clientContext.getYarnConfig(); + } +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java new file mode 100644 index 0000000000000000000000000000000000000000..4353e3dc2b051c0f5ccb631b9f6b79bf637ab934 --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli; + +import org.apache.hadoop.conf.Configuration; +import org.apache.submarine.client.cli.remote.RpcRuntimeFactory; +import org.apache.submarine.client.cli.runjob.RunJobCli; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.submarine.commons.runtime.ClientContext; +import org.apache.submarine.commons.runtime.RuntimeFactory; +import org.apache.submarine.commons.utils.SubmarineConfVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +public class Cli { + private static final Logger LOG = + LoggerFactory.getLogger(Cli.class); + + private static void printHelp() { + StringBuilder helpMsg = new StringBuilder(); + helpMsg.append("\n\nUsage: [] []\n"); + helpMsg.append(" Below are all objects / actions:\n"); + helpMsg.append(" job \n"); + helpMsg.append(" run : run a job, please see 'job run --help' for usage \n"); + helpMsg.append(" show : get status of job, please see 'job show --help' for usage \n"); + + System.out.println(helpMsg.toString()); + } + + private static ClientContext getClientContext() { + ClientContext clientContext = new ClientContext(); + RuntimeFactory runtimeFactory; + if (clientContext.getSubmarineConfig().getBoolean( + SubmarineConfVars.ConfVars. + SUBMARINE_SERVER_RPC_ENABLED)) { + runtimeFactory = new RpcRuntimeFactory(clientContext); + } else { + Configuration conf = new YarnConfiguration(); + clientContext.setYarnConfig(conf); + runtimeFactory = RuntimeFactory.getRuntimeFactory(clientContext, + Thread.currentThread().getContextClassLoader()); + } + clientContext.setRuntimeFactory(runtimeFactory); + return clientContext; + } + + public static void main(String[] args) throws Exception { + System.out.println(" _ _ \n" + + " | | (_) \n" + + " ___ _ _ | |__ _ __ ___ __ _ _ __ _ _ __ ___ \n" + + " / __|| | | || '_ \\ | '_ ` _ \\ / _` || '__|| || '_ \\ / _ \\\n" + + " \\__ \\| |_| || |_) || | | | | || (_| || | | || | | || __/\n" + + " |___/ \\__,_||_.__/ |_| |_| |_| \\__,_||_| |_||_| |_| \\___|\n" + + " \n" + + " ?\n" + + " ~~~~~~~~~~~~~~~~~~~~~~~~~~~|^\"~~~~~~~~~~~~~~~~~~~~~~~~~o~~~~~~~~~~~\n" + + " o | o __o\n" + + " o | o |X__>\n" + + " ___o | __o\n" + + " (X___>-- __|__ |X__> o\n" + + " | \\ __o\n" + + " | \\ |X__>\n" + + " _______________________|_______\\________________\n" + + " < \\____________ _\n" + + " \\ \\ (_)\n" + + " \\ O O O >=)\n" + + " \\__________________________________________________________/ (_)\n" + + "\n"); + + if (CliUtils.argsForHelp(args)) { + printHelp(); + System.exit(0); + } + + if (args.length < 2) { + LOG.error("Bad parameters specified."); + printHelp(); + System.exit(-1); + } + + String[] moduleArgs = Arrays.copyOfRange(args, 2, args.length); + ClientContext clientContext = getClientContext(); + + if (args[0].equals("job")) { + String subCmd = args[1]; + if (subCmd.equals(CliConstants.RUN)) { + new RunJobCli(clientContext).run(moduleArgs); + } else if (subCmd.equals(CliConstants.SHOW)) { + new ShowJobCli(clientContext).run(moduleArgs); + } else { + printHelp(); + throw new IllegalArgumentException("Unknown option for job"); + } + } else { + printHelp(); + throw new IllegalArgumentException("Bad parameters "); + } + } +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/CliConstants.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/CliConstants.java new file mode 100644 index 0000000000000000000000000000000000000000..e89ad1b78964e58c6c717004dd8ac89e69bb56ca --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/CliConstants.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli; + +/* + * NOTE: use lowercase + "_" for the option name + */ +public class CliConstants { + public static final String RUN = "run"; + public static final String SERVE = "serve"; + public static final String LIST = "list"; + public static final String SHOW = "show"; + public static final String NAME = "name"; + public static final String INPUT_PATH = "input_path"; + public static final String CHECKPOINT_PATH = "checkpoint_path"; + public static final String SAVED_MODEL_PATH = "saved_model_path"; + public static final String N_WORKERS = "num_workers"; + public static final String N_SERVING_TASKS = "num_serving_tasks"; + public static final String N_PS = "num_ps"; + public static final String WORKER_RES = "worker_resources"; + public static final String SERVING_RES = "serving_resources"; + public static final String PS_RES = "ps_resources"; + public static final String SCHEDULER_RES = "scheduler_resources"; + public static final String N_SCHEDULERS = "num_schedulers"; + public static final String DOCKER_IMAGE = "docker_image"; + public static final String QUEUE = "queue"; + public static final String TENSORBOARD = "tensorboard"; + public static final String TENSORBOARD_RESOURCES = "tensorboard_resources"; + public static final String TENSORBOARD_DEFAULT_RESOURCES = + "memory=4G,vcores=1"; + public static final String ARG_CONF = "conf"; + + public static final String WORKER_LAUNCH_CMD = "worker_launch_cmd"; + public static final String SERVING_LAUNCH_CMD = "serving_launch_cmd"; + public static final String PS_LAUNCH_CMD = "ps_launch_cmd"; + public static final String SCHEDULER_LAUNCH_CMD = "scheduler_launch_cmd"; + public static final String ENV = "env"; + public static final String VERBOSE = "verbose"; + public static final String SERVING_FRAMEWORK = "serving_framework"; + public static final String STOP = "stop"; + public static final String WAIT_JOB_FINISH = "wait_job_finish"; + public static final String PS_DOCKER_IMAGE = "ps_docker_image"; + public static final String WORKER_DOCKER_IMAGE = "worker_docker_image"; + public static final String SCHEDULER_DOCKER_IMAGE = "scheduler_docker_image"; + public static final String QUICKLINK = "quicklink"; + public static final String TENSORBOARD_DOCKER_IMAGE = + "tensorboard_docker_image"; + public static final String LOCALIZATION = "localization"; + public static final String KEYTAB = "keytab"; + public static final String PRINCIPAL = "principal"; + public static final String DISTRIBUTE_KEYTAB = "distribute_keytab"; + public static final String YAML_CONFIG = "f"; + public static final String INSECURE_CLUSTER = "insecure"; + + public static final String FRAMEWORK = "framework"; +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/CliUtils.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/CliUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..222de1f634e75d3f5d5cbe5cedfa1fa235f03dc6 --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/CliUtils.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.submarine.client.cli.param.runjob.RunJobParameters; +import org.apache.submarine.commons.runtime.exception.SubmarineRuntimeException; +import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class CliUtils { + private static final Logger LOG = + LoggerFactory.getLogger(CliUtils.class); + + /** + * Replace patterns inside cli + * + * @return launch command after pattern replace + */ + public static String replacePatternsInLaunchCommand(String specifiedCli, + RunJobParameters jobRunParameters, + RemoteDirectoryManager directoryManager) + throws IOException { + String input = jobRunParameters.getInputPath(); + String jobDir = jobRunParameters.getCheckpointPath(); + String savedModelDir = jobRunParameters.getSavedModelPath(); + + Map replacePattern = new HashMap<>(); + if (jobDir != null) { + replacePattern.put("%" + CliConstants.CHECKPOINT_PATH + "%", jobDir); + } + if (input != null) { + replacePattern.put("%" + CliConstants.INPUT_PATH + "%", input); + } + if (savedModelDir != null) { + replacePattern.put("%" + CliConstants.SAVED_MODEL_PATH + "%", + savedModelDir); + } + + String newCli = specifiedCli; + for (Map.Entry replace : replacePattern.entrySet()) { + newCli = newCli.replace(replace.getKey(), replace.getValue()); + } + + return newCli; + } + + // Is it for help? + public static boolean argsForHelp(String[] args) { + if (args == null || args.length == 0) + return true; + + if (args.length == 1) { + return args[0].equals("-h") || args[0].equals("--help"); + } + + return false; + } + + public static void doLoginIfSecure(String keytab, String principal) throws + IOException { + if (!UserGroupInformation.isSecurityEnabled()) { + return; + } + + if (StringUtils.isEmpty(keytab) || StringUtils.isEmpty(principal)) { + if (StringUtils.isNotEmpty(keytab)) { + SubmarineRuntimeException e = new SubmarineRuntimeException("The " + + "parameter of " + CliConstants.PRINCIPAL + " is missing."); + LOG.error(e.getMessage(), e); + throw e; + } + + if (StringUtils.isNotEmpty(principal)) { + SubmarineRuntimeException e = new SubmarineRuntimeException("The " + + "parameter of " + CliConstants.KEYTAB + " is missing."); + LOG.error(e.getMessage(), e); + throw e; + } + + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + if (user == null || user.getAuthenticationMethod() == + UserGroupInformation.AuthenticationMethod.SIMPLE) { + SubmarineRuntimeException e = new SubmarineRuntimeException("Failed " + + "to authenticate in secure environment. Please run kinit " + + "command in advance or use " + "--" + CliConstants.KEYTAB + "/--" + CliConstants.PRINCIPAL + + " parameters"); + LOG.error(e.getMessage(), e); + throw e; + } + LOG.info("Submarine job is submitted by user: " + user.getUserName()); + return; + } + + File keytabFile = new File(keytab); + if (!keytabFile.exists()) { + SubmarineRuntimeException e = new SubmarineRuntimeException("No " + + "keytab localized at " + keytab); + LOG.error(e.getMessage(), e); + throw e; + } + UserGroupInformation.loginUserFromKeytab(principal, keytab); + } + + /** + * As hadoop-2.7 doesn't have this method, we add this method in submarine. + * @param appIdStr + * @return + */ + public static ApplicationId fromString(String appIdStr) { + String APPLICATION_ID_PREFIX = "application_"; + if (!appIdStr.startsWith(APPLICATION_ID_PREFIX)) { + throw new IllegalArgumentException("Invalid ApplicationId prefix: " + + appIdStr + ". The valid ApplicationId should start with prefix " + + "application"); + } + try { + int pos1 = APPLICATION_ID_PREFIX.length() - 1; + int pos2 = appIdStr.indexOf('_', pos1 + 1); + if (pos2 < 0) { + throw new IllegalArgumentException("Invalid ApplicationId: " + + appIdStr); + } + long rmId = Long.parseLong(appIdStr.substring(pos1 + 1, pos2)); + int appId = Integer.parseInt(appIdStr.substring(pos2 + 1)); + ApplicationId applicationId = ApplicationId.newInstance(rmId, appId); + return applicationId; + } catch (NumberFormatException n) { + throw new IllegalArgumentException("Invalid ApplicationId: " + + appIdStr, n); + } + } +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/Command.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/Command.java new file mode 100644 index 0000000000000000000000000000000000000000..f19cd54a35672353f80a4f4347ec40b5777d00fe --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/Command.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli; + +/** + * Represents a Submarine command. + */ +public enum Command { + RUN_JOB, SHOW_JOB +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/ShowJobCli.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/ShowJobCli.java new file mode 100644 index 0000000000000000000000000000000000000000..651a542e469d277aed4dbfd9d58d8331e5f865d9 --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/ShowJobCli.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.submarine.client.cli.param.ParametersHolder; +import org.apache.submarine.client.cli.param.ShowJobParameters; +import org.apache.submarine.commons.runtime.ClientContext; +import org.apache.submarine.commons.runtime.exception.SubmarineException; +import org.apache.submarine.commons.runtime.fs.StorageKeyConstants; +import org.apache.submarine.commons.runtime.fs.SubmarineStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +public class ShowJobCli extends AbstractCli { + private static final Logger LOG = LoggerFactory.getLogger(ShowJobCli.class); + + private Options options; + private ParametersHolder parametersHolder; + + public ShowJobCli(ClientContext cliContext) { + super(cliContext); + options = generateOptions(); + } + + public void printUsages() { + new HelpFormatter().printHelp("job show", options); + } + + private Options generateOptions() { + Options options = new Options(); + options.addOption(CliConstants.NAME, true, "Name of the job"); + options.addOption("h", "help", false, "Print help"); + return options; + } + + private void parseCommandLineAndGetShowJobParameters(String[] args) + throws IOException, YarnException { + // Do parsing + GnuParser parser = new GnuParser(); + CommandLine cli; + try { + cli = parser.parse(options, args); + parametersHolder = ParametersHolder + .createWithCmdLine(cli, Command.SHOW_JOB); + parametersHolder.updateParameters(clientContext); + } catch (ParseException e) { + printUsages(); + } + } + + private void printIfNotNull(String keyForPrint, String keyInStorage, + Map jobInfo) { + if (jobInfo.containsKey(keyInStorage)) { + System.out.println("\t" + keyForPrint + ": " + jobInfo.get(keyInStorage)); + } + } + + private void printJobInfo(Map jobInfo) { + System.out.println("Job Meta Info:"); + printIfNotNull("Application Id", StorageKeyConstants.APPLICATION_ID, + jobInfo); + printIfNotNull("Input Path", StorageKeyConstants.INPUT_PATH, jobInfo); + printIfNotNull("Saved Model Path", StorageKeyConstants.SAVED_MODEL_PATH, + jobInfo); + printIfNotNull("Checkpoint Path", StorageKeyConstants.CHECKPOINT_PATH, + jobInfo); + printIfNotNull("Run Parameters", StorageKeyConstants.JOB_RUN_ARGS, + jobInfo); + } + + @VisibleForTesting + protected void getAndPrintJobInfo() throws IOException { + SubmarineStorage storage = + clientContext.getRuntimeFactory().getSubmarineStorage(); + + Map jobInfo = null; + try { + jobInfo = storage.getJobInfoByName(getParameters().getName()); + } catch (IOException e) { + LOG.error("Failed to retrieve job info", e); + throw e; + } + + printJobInfo(jobInfo); + } + + @VisibleForTesting + public ShowJobParameters getParameters() { + return (ShowJobParameters) parametersHolder.getParameters(); + } + + @Override + public int run(String[] args) + throws ParseException, IOException, YarnException, InterruptedException, + SubmarineException { + if (CliUtils.argsForHelp(args)) { + printUsages(); + return 0; + } + parseCommandLineAndGetShowJobParameters(args); + getAndPrintJobInfo(); + return 0; + } +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/param/ConfigType.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/ConfigType.java new file mode 100644 index 0000000000000000000000000000000000000000..e5996d14ab547d90f1e9884089186e55709a6d53 --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/ConfigType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli.param; + +/** + * Represents the source of configuration. + */ +public enum ConfigType { + YAML, CLI +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/param/Localization.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/Localization.java new file mode 100644 index 0000000000000000000000000000000000000000..03774504140fa3e6f7061f81a5eed95cffaf00f7 --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/Localization.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli.param; + +import org.apache.commons.cli.ParseException; + +import java.util.Arrays; +import java.util.List; + +/** + * Localization parameter. + * */ +public class Localization { + + private String mountPermissionPattern = "(wr|rw)$"; + /** + * Regex for directory/file path in container. + * YARN only support absolute path for mount, but we can + * support some relative path. + * For relative path, we only allow ".", "./","./name". + * relative path like "./a/b" is not allowed. + * "." and "./" means original dir/file name in container working directory + * "./name" means use same or new "name" in container working directory + * A absolute path means same path in container filesystem + */ + private String localPathPattern = "((^\\.$)|(^\\./$)|(^\\./[^/]+)|(^/.*))"; + private String remoteUri; + private String localPath; + + // Read write by default + private String mountPermission = "rw"; + + private static final List SUPPORTED_SCHEME = Arrays.asList( + "hdfs", "oss", "s3a", "s3n", "wasb", + "wasbs", "abfs", "abfss", "adl", "har", + "ftp", "http", "https", "viewfs", "swebhdfs", + "webhdfs", "swift"); + + public void parse(String arg) throws ParseException { + String[] tokens = arg.split(":"); + int minimum = "a:b".split(":").length; + int minimumWithPermission = "a:b:rw".split(":").length; + int minimumParts = minimum; + int miniPartsWithRemoteScheme = "scheme://a:b".split(":").length; + int maximumParts = "scheme://a:b:rw".split(":").length; + // If remote uri starts with a remote scheme + if (isSupportedScheme(tokens[0])) { + minimumParts = miniPartsWithRemoteScheme; + } + if (tokens.length < minimumParts + || tokens.length > maximumParts) { + throw new ParseException("Invalid parameter," + + "should be \"remoteUri:localPath[:rw|:wr]\" " + + "format for --localizations"); + } + + /** + * RemoteUri starts with remote scheme. + * Merge part 0 and 1 to build a hdfs path in token[0]. + * toke[1] will be localPath to ease following logic + * */ + if (minimumParts == miniPartsWithRemoteScheme) { + tokens[0] = tokens[0] + ":" + tokens[1]; + tokens[1] = tokens[2]; + if (tokens.length == maximumParts) { + // Has permission part + mountPermission = tokens[maximumParts - 1]; + } + } + // RemoteUri starts with linux file path + if (minimumParts == minimum + && tokens.length == minimumWithPermission) { + // Has permission part + mountPermission = tokens[minimumWithPermission - 1]; + } + remoteUri = tokens[0]; + localPath = tokens[1]; + if (!localPath.matches(localPathPattern)) { + throw new ParseException("Invalid local file path:" + + localPath + + ", it only support \".\", \"./\", \"./name\" and " + + "absolute path."); + } + if (!mountPermission.matches(mountPermissionPattern)) { + throw new ParseException("Invalid mount permission (ro is not " + + "supported yet), " + mountPermission); + } + } + + public String getRemoteUri() { + return remoteUri; + } + + public Localization setRemoteUri(String rUti) { + this.remoteUri = rUti; + return this; + } + + public String getLocalPath() { + return localPath; + } + + public Localization setLocalPath(String lPath) { + this.localPath = lPath; + return this; + } + + public String getMountPermission() { + return mountPermission; + } + + public Localization setMountPermission(String mPermission) { + this.mountPermission = mPermission; + return this; + } + + private boolean isSupportedScheme(String scheme) { + return SUPPORTED_SCHEME.contains(scheme); + } +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/param/ParametersHolder.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/ParametersHolder.java new file mode 100644 index 0000000000000000000000000000000000000000..556dfe6e11f93d091bc2aa501b025d7b7005294c --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/ParametersHolder.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli.param; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.ParseException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.submarine.client.cli.CliConstants; +import org.apache.submarine.client.cli.Command; +import org.apache.submarine.client.cli.param.runjob.MXNetRunJobParameters; +import org.apache.submarine.client.cli.param.runjob.PyTorchRunJobParameters; +import org.apache.submarine.client.cli.param.runjob.TensorFlowRunJobParameters; +import org.apache.submarine.client.cli.param.yaml.Configs; +import org.apache.submarine.client.cli.param.yaml.Role; +import org.apache.submarine.client.cli.param.yaml.Roles; +import org.apache.submarine.client.cli.param.yaml.Scheduling; +import org.apache.submarine.client.cli.param.yaml.Security; +import org.apache.submarine.client.cli.param.yaml.TensorBoard; +import org.apache.submarine.client.cli.param.yaml.YamlConfigFile; +import org.apache.submarine.client.cli.param.yaml.YamlParseException; +import org.apache.submarine.commons.runtime.param.BaseParameters; +import org.apache.submarine.commons.runtime.Framework; +import org.apache.submarine.commons.runtime.ClientContext; +import org.apache.submarine.commons.runtime.param.Parameter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.submarine.client.cli.runjob.RunJobCli.YAML_PARSE_FAILED; + +/** + * This class acts as a wrapper of {@code CommandLine} values along with + * YAML configuration values. + * YAML configuration is only stored if the -f <filename> + * option is specified along the CLI arguments. + * Using this wrapper class makes easy to deal with + * any form of configuration source potentially added into Submarine, + * in the future. + * If both YAML and CLI value is found for a config, this is an error case. + */ +public final class ParametersHolder implements Parameter { + private static final Logger LOG = + LoggerFactory.getLogger(ParametersHolder.class); + + public static final String SUPPORTED_FRAMEWORKS_MESSAGE = + "TensorFlow, PyTorch, MXNet are the only supported frameworks for now!"; + public static final String SUPPORTED_COMMANDS_MESSAGE = + "'Show job' and 'run job' are the only supported commands for now!"; + + + + private CommandLine parsedCommandLine; + private Map yamlStringConfigs; + private Map> yamlListConfigs; + private ConfigType configType; + private Command command; + private final Set onlyDefinedWithCliArgs = ImmutableSet.of( + CliConstants.VERBOSE); + private Framework framework; + private BaseParameters parameters; + + private ParametersHolder(CommandLine parsedCommandLine, + YamlConfigFile yamlConfig, ConfigType configType, Command command) + throws ParseException, YarnException { + this.parsedCommandLine = parsedCommandLine; + this.yamlStringConfigs = initStringConfigValues(yamlConfig); + this.yamlListConfigs = initListConfigValues(yamlConfig); + this.configType = configType; + this.command = command; + this.framework = determineFrameworkType(); + this.ensureOnlyValidSectionsAreDefined(yamlConfig); + this.parameters = createParameters(); + } + + private ParametersHolder(){ + super(); + } + + private BaseParameters createParameters() { + if (command == Command.RUN_JOB) { + if (framework == Framework.TENSORFLOW) { + return new TensorFlowRunJobParameters(); + } else if (framework == Framework.PYTORCH) { + return new PyTorchRunJobParameters(); + } else if (framework == Framework.MXNET) { + return new MXNetRunJobParameters(); + } else { + throw new UnsupportedOperationException(SUPPORTED_FRAMEWORKS_MESSAGE); + } + } else if (command == Command.SHOW_JOB) { + return new ShowJobParameters(); + } else { + throw new UnsupportedOperationException(SUPPORTED_COMMANDS_MESSAGE); + } + } + + private void ensureOnlyValidSectionsAreDefined(YamlConfigFile yamlConfig) { + if (isCommandRunJob() && isFrameworkPyTorch() && + isPsSectionDefined(yamlConfig)) { + throw new YamlParseException( + "PS section should not be defined when PyTorch " + + "is the selected framework!"); + } + + if (isCommandRunJob() && (isFrameworkPyTorch() || isFrameworkMXNet()) + && isTensorboardSectionDefined(yamlConfig)) { + throw new YamlParseException( + "TensorBoard section should not be defined when TensorFlow " + + "is not the selected framework!"); + } + + if (isCommandRunJob() && !isFrameworkMXNet() && + isSchedulerSectionDefined(yamlConfig)) { + throw new YamlParseException( + "Scheduler section should not be defined when MXNet " + + "is not the selected framework!"); + } + } + + private boolean isCommandRunJob() { + return command == Command.RUN_JOB; + } + + private boolean isFrameworkPyTorch() { + return framework == Framework.PYTORCH; + } + + private boolean isFrameworkMXNet() { + return framework == Framework.MXNET; + } + + private boolean isPsSectionDefined(YamlConfigFile yamlConfig) { + return yamlConfig != null && + yamlConfig.getRoles() != null && + yamlConfig.getRoles().getPs() != null; + } + + private boolean isTensorboardSectionDefined(YamlConfigFile yamlConfig) { + return yamlConfig != null && + yamlConfig.getTensorBoard() != null; + } + + private boolean isSchedulerSectionDefined(YamlConfigFile yamlConfig) { + return yamlConfig != null && + yamlConfig.getRoles() != null && + yamlConfig.getRoles().getScheduler() != null; + } + + private Framework determineFrameworkType() + throws ParseException, YarnException { + if (!isCommandRunJob()) { + return null; + } + String frameworkStr = getOptionValue(CliConstants.FRAMEWORK); + if (frameworkStr == null) { + LOG.info("Framework is not defined in config, falling back to " + + "TensorFlow as a default."); + return Framework.TENSORFLOW; + } + Framework framework = Framework.parseByValue(frameworkStr); + if (framework == null) { + if (getConfigType() == ConfigType.CLI) { + throw new ParseException("Failed to parse Framework type! " + + "Valid values are: " + Framework.getValues()); + } else { + throw new YamlParseException(YAML_PARSE_FAILED + + ", framework should is defined, but it has an invalid value! " + + "Valid values are: " + Framework.getValues()); + } + } + return framework; + } + + /** + * Maps every value coming from the passed yamlConfig to {@code CliConstants}. + * @param yamlConfig Parsed YAML config + * @return A map of config values, keys are {@code CliConstants} + * and values are Strings. + */ + private Map initStringConfigValues( + YamlConfigFile yamlConfig) { + if (yamlConfig == null) { + return Collections.emptyMap(); + } + Map yamlConfigValues = Maps.newHashMap(); + Roles roles = yamlConfig.getRoles(); + + initGenericConfigs(yamlConfig, yamlConfigValues); + initPs(yamlConfigValues, roles.getPs()); + initWorker(yamlConfigValues, roles.getWorker()); + initScheduler(yamlConfigValues, roles.getScheduler()); + initScheduling(yamlConfigValues, yamlConfig.getScheduling()); + initSecurity(yamlConfigValues, yamlConfig.getSecurity()); + initTensorBoard(yamlConfigValues, yamlConfig.getTensorBoard()); + + return yamlConfigValues; + } + + private Map> initListConfigValues( + YamlConfigFile yamlConfig) { + if (yamlConfig == null) { + return Collections.emptyMap(); + } + + Map> yamlConfigValues = Maps.newHashMap(); + Configs configs = yamlConfig.getConfigs(); + yamlConfigValues.put(CliConstants.LOCALIZATION, configs.getLocalizations()); + yamlConfigValues.put(CliConstants.ENV, + convertToEnvsList(configs.getEnvs())); + yamlConfigValues.put(CliConstants.QUICKLINK, configs.getQuicklinks()); + + return yamlConfigValues; + } + + private void initGenericConfigs(YamlConfigFile yamlConfig, + Map yamlConfigs) { + yamlConfigs.put(CliConstants.NAME, yamlConfig.getSpec().getName()); + yamlConfigs.put(CliConstants.FRAMEWORK, + yamlConfig.getSpec().getFramework()); + + Configs configs = yamlConfig.getConfigs(); + yamlConfigs.put(CliConstants.INPUT_PATH, configs.getInputPath()); + yamlConfigs.put(CliConstants.CHECKPOINT_PATH, configs.getCheckpointPath()); + yamlConfigs.put(CliConstants.SAVED_MODEL_PATH, configs.getSavedModelPath()); + yamlConfigs.put(CliConstants.DOCKER_IMAGE, configs.getDockerImage()); + yamlConfigs.put(CliConstants.WAIT_JOB_FINISH, configs.getWaitJobFinish()); + } + + private void initPs(Map yamlConfigs, Role ps) { + if (ps == null) { + return; + } + yamlConfigs.put(CliConstants.N_PS, String.valueOf(ps.getReplicas())); + yamlConfigs.put(CliConstants.PS_RES, ps.getResources()); + yamlConfigs.put(CliConstants.PS_DOCKER_IMAGE, ps.getDockerImage()); + yamlConfigs.put(CliConstants.PS_LAUNCH_CMD, ps.getLaunchCmd()); + } + + private void initWorker(Map yamlConfigs, Role worker) { + if (worker == null) { + return; + } + yamlConfigs.put(CliConstants.N_WORKERS, + String.valueOf(worker.getReplicas())); + yamlConfigs.put(CliConstants.WORKER_RES, worker.getResources()); + yamlConfigs.put(CliConstants.WORKER_DOCKER_IMAGE, worker.getDockerImage()); + yamlConfigs.put(CliConstants.WORKER_LAUNCH_CMD, worker.getLaunchCmd()); + } + + private void initScheduler(Map yamlConfigs, Role scheduler) { + if (scheduler == null) { + return; + } + yamlConfigs.put(CliConstants.N_SCHEDULERS, + String.valueOf(scheduler.getReplicas())); + yamlConfigs.put(CliConstants.SCHEDULER_RES, scheduler.getResources()); + yamlConfigs.put(CliConstants.SCHEDULER_DOCKER_IMAGE, scheduler.getDockerImage()); + yamlConfigs.put(CliConstants.SCHEDULER_LAUNCH_CMD, scheduler.getLaunchCmd()); + } + + private void initScheduling(Map yamlConfigValues, + Scheduling scheduling) { + if (scheduling == null) { + return; + } + yamlConfigValues.put(CliConstants.QUEUE, scheduling.getQueue()); + } + + private void initSecurity(Map yamlConfigValues, + Security security) { + if (security == null) { + return; + } + yamlConfigValues.put(CliConstants.KEYTAB, security.getKeytab()); + yamlConfigValues.put(CliConstants.PRINCIPAL, security.getPrincipal()); + yamlConfigValues.put(CliConstants.DISTRIBUTE_KEYTAB, + String.valueOf(security.isDistributeKeytab())); + } + + private void initTensorBoard(Map yamlConfigValues, + TensorBoard tensorBoard) { + if (tensorBoard == null) { + return; + } + yamlConfigValues.put(CliConstants.TENSORBOARD, Boolean.TRUE.toString()); + yamlConfigValues.put(CliConstants.TENSORBOARD_DOCKER_IMAGE, + tensorBoard.getDockerImage()); + yamlConfigValues.put(CliConstants.TENSORBOARD_RESOURCES, + tensorBoard.getResources()); + } + + private List convertToEnvsList(Map envs) { + if (envs == null) { + return Collections.emptyList(); + } + return envs.entrySet().stream() + .map(e -> String.format("%s=%s", e.getKey(), e.getValue())) + .collect(Collectors.toList()); + } + + public static ParametersHolder create() { + return new ParametersHolder(); + } + + public static ParametersHolder createWithCmdLine(CommandLine cli, + Command command) throws ParseException, YarnException { + return new ParametersHolder(cli, null, ConfigType.CLI, command); + } + + public static ParametersHolder createWithCmdLineAndYaml(CommandLine cli, + YamlConfigFile yamlConfig, Command command) throws ParseException, + YarnException { + return new ParametersHolder(cli, yamlConfig, ConfigType.YAML, command); + } + + /** + * Gets the option value, either from the CLI arguments or YAML config, + * if present. + * @param option Name of the config. + * @return The value of the config + */ + public String getOptionValue(String option) throws YarnException { + ensureConfigIsDefinedOnce(option, true); + if (onlyDefinedWithCliArgs.contains(option) || + parsedCommandLine.hasOption(option)) { + return getValueFromCLI(option); + } + return getValueFromYaml(option); + } + + /** + * Gets the option values, either from the CLI arguments or YAML config, + * if present. + * @param option Name of the config. + * @return The values of the config + */ + public List getOptionValues(String option) throws YarnException { + ensureConfigIsDefinedOnce(option, false); + if (onlyDefinedWithCliArgs.contains(option) || + parsedCommandLine.hasOption(option)) { + return getValuesFromCLI(option); + } + return getValuesFromYaml(option); + } + + private void ensureConfigIsDefinedOnce(String option, boolean stringValue) + throws YarnException { + boolean definedWithYaml; + if (stringValue) { + definedWithYaml = yamlStringConfigs.containsKey(option); + } else { + definedWithYaml = yamlListConfigs.containsKey(option); + } + + if (parsedCommandLine.hasOption(option) && definedWithYaml) { + throw new YarnException("Config '%s' is defined both with YAML config" + + " and with CLI argument, please only use either way!"); + } + } + + private String getValueFromCLI(String option) { + String value = parsedCommandLine.getOptionValue(option); + if (LOG.isDebugEnabled()) { + LOG.debug("Found config value {} for key {} " + + "from CLI configuration.", value, option); + } + return value; + } + + private List getValuesFromCLI(String option) { + String[] optionValues = parsedCommandLine.getOptionValues(option); + if (optionValues != null) { + List values = Arrays.asList(optionValues); + if (LOG.isDebugEnabled()) { + LOG.debug("Found config values {} for key {} " + + "from CLI configuration.", values, option); + } + return values; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No config values found for key {} " + + "from CLI configuration.", option); + } + return Lists.newArrayList(); + } + } + + private String getValueFromYaml(String option) { + String value = yamlStringConfigs.get(option); + if (LOG.isDebugEnabled()) { + LOG.debug("Found config value {} for key {} " + + "from YAML configuration.", value, option); + } + return value; + } + + private List getValuesFromYaml(String option) { + List values = yamlListConfigs.get(option); + if (LOG.isDebugEnabled()) { + LOG.debug("Found config values {} for key {} " + + "from YAML configuration.", values, option); + } + return values; + } + + /** + * Returns the boolean value of option. + * First, we check if the CLI value is defined for the option. + * If not, then we check the YAML value. + * @param option name of the option + * @return true, if the option is found in the CLI args or in the YAML config, + * false otherwise. + */ + public boolean hasOption(String option) { + if (onlyDefinedWithCliArgs.contains(option)) { + boolean value = parsedCommandLine.hasOption(option); + if (LOG.isDebugEnabled()) { + LOG.debug("Found boolean config with value {} for key {} " + + "from CLI configuration.", value, option); + } + return value; + } + if (parsedCommandLine.hasOption(option)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Found boolean config value for key {} " + + "from CLI configuration.", option); + } + return true; + } + return getBooleanValueFromYaml(option); + } + + private boolean getBooleanValueFromYaml(String option) { + String stringValue = yamlStringConfigs.get(option); + boolean result = stringValue != null + && Boolean.valueOf(stringValue).equals(Boolean.TRUE); + LOG.debug("Found config value {} for key {} " + + "from YAML configuration.", result, option); + return result; + } + + public ConfigType getConfigType() { + return configType; + } + + public Framework getFramework() { + return framework; + } + + @Override + public Parameter setFramework(Framework framework) { + this.framework = framework; + return this; + } + + public void updateParameters(ClientContext clientContext) + throws ParseException, YarnException, IOException { + parameters.updateParameters(this, clientContext); + } + + public BaseParameters getParameters() { + return parameters; + } + + public Parameter setParameters(BaseParameters parameters) { + this.parameters = parameters; + return this; + } + + public CommandLine getParsedCommandLine() { + return parsedCommandLine; + } + + public Parameter setParsedCommandLine(CommandLine parsedCommandLine) { + this.parsedCommandLine = parsedCommandLine; + return this; + } + + public Map getYamlStringConfigs() { + return yamlStringConfigs; + } + + public Parameter setYamlStringConfigs(Map yamlStringConfigs) { + this.yamlStringConfigs = yamlStringConfigs; + return this; + } + + public Map> getYamlListConfigs() { + return yamlListConfigs; + } + + public Parameter setYamlListConfigs( + Map> yamlListConfigs) { + this.yamlListConfigs = yamlListConfigs; + return this; + } +} diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/param/Quicklink.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/Quicklink.java new file mode 100644 index 0000000000000000000000000000000000000000..4fe035c4ead4a05f4c57079728454de044bf00b7 --- /dev/null +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/param/Quicklink.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.client.cli.param; + +import org.apache.commons.cli.ParseException; + +/** + * A class represents quick links to a web page. + */ +public class Quicklink { + private String label; + private String componentInstanceName; + private String protocol; + private int port; + + public void parse(String quicklinkStr) throws ParseException { + if (!quicklinkStr.contains("=")) { + throw new ParseException("Should be