Руководство по установке#

Defaults:<user> !requiretty

units
├── global_service_vars.yml
└── my_unit
    ├── clusters
    │   └── my_cluster
    │       ├── devops.yml
    │       ├── global_service_vars.yml
    │       ├── hosts.yml
    │       ├── hosts_resolve.yml
    │       ├── pass.yml
    │       ├── service_descriptors
    │       │   ├── service.yml
    │       │   └── ...
    │       └── service_mapping.yml
    ├── global_service_vars.yml
    ├── service_descriptors
    │   ├── service.yml
    │   └── ...
    └── unit.yml

service_id: service_name
service_type: SERVICE_TYPE
meta:
  role_N:
    cardinality: 1+
logic_config_group_1:
  param_1_logic_config_group_1:
    default: null
    required: false
...
  param_N_logic_config_group_1:
    default: true
    required: false
...
logic_config_group_N:
  param_1_logic_config_group_N:
    default: null
    required: false
    ...
  param_N_logic_config_group_N:
    default: true
    required: false

service_id: service_name

service_type: SERVICE_TYPE

meta:
  role_1:
    cardinality: 1+
  ...
  role_N:
	cardinality: 1+

logic_config_group_1:
  param_1_logic_config_group_1:
    default: null
    required: false
	  type: string
	...
  param_N_logic_config_group_1:
    default: true
    required: false
	  type: string
...
role_N:
  param_1_logic_config_group_N:
    default: null
    required: false
	  type: string
	...
  param_N_logic_config_group_N:
    default: true
    required: false
	  type: string

service_id: ARCHIVE_SERVICE
service_type: ARCHIVE_SERVICE
meta:
  archive:
    cardinality: 1+
archive-service-config:
  abyss.audit.discovery.zookeeper.path:
    default: /abyss/audit/metamodel
    sensitive: false
    required: true
  abyss.audit.event.publish-url:
    default: ''
    sensitive: false
    required: true
  common.actuator.basic-auth.enabled:
    default: 'false'
    sensitive: false
    required: true
  common.actuator.basic-auth.password:
    default: null
    sensitive: true
    required: true
  common.actuator.basic-auth.user:
    default: ''
    sensitive: false
    required: true
  common.auth.certificate-authentication-enabled:
    default: 'true'
    sensitive: false
    required: true
  common.auth.certificate.get-cert-from-header:
    default: 'true'
    sensitive: false
    required: true
  common.auth.jwk-resource-server:
    default: ''
    sensitive: false
    required: true
  common.auth.secret:
    default: null
    sensitive: true
    required: true
  common.auth.type:
    default: SECRET_AND_JWK
    sensitive: false
    required: true
  common.discovery.type:
    default: ZOOKEEPER
    sensitive: false
    required: true
  common.discovery.zookeeper.connection:
    default: ''
    sensitive: false
    required: true
  common.discovery.zookeeper.leadership-enabled:
    default: 'true'
    sensitive: false
    required: true
  common.discovery.zookeeper.path:
    default: /abyss
    sensitive: false
    required: true
  common.discovery.zookeeper.retry-interval-ms:
    default: '10000'
    sensitive: false
    required: true
  common.global.client.ssl.key-password:
    default: null
    sensitive: true
    required: true
  common.global.client.ssl.keystore-location:
    default: ''
    sensitive: false
    required: true
  common.global.client.ssl.keystore-password:
    default: null
    sensitive: true
    required: true
  common.global.client.ssl.store-type:
    default: JKS
    sensitive: false
    required: true
  common.global.client.ssl.truststore-location:
    default: ''
    sensitive: false
    required: true
  common.global.client.ssl.truststore-password:
    default: null
    sensitive: true
    required: true
  common.global.client.ssl.verify-hosts:
    default: 'false'
    sensitive: false
    required: true
  common.http.client.connect-timeout-ms:
    default: '50000'
    sensitive: false
    required: true
  common.http.client.eviction.idle-sec:
    default: '10000'
    sensitive: false
    required: true
  common.http.client.eviction.interval-ms:
    default: '10000'
    sensitive: false
    required: true
  common.http.client.keep-alive-ms:
    default: '10000'
    sensitive: false
    required: true
  common.http.client.max-connections:
    default: '1000'
    sensitive: false
    required: true
  common.http.client.socket-timeout-ms:
    default: '50000'
    sensitive: false
    required: true
  common.internal-communication.data-center-id:
    default: ''
    sensitive: false
    required: true
  common.internal-communication.max-retries:
    default: '2'
    sensitive: false
    required: true
  common.internal-communication.ssl.protocol:
    default: PLAINTEXT
    sensitive: false
    required: true
  common.pvm-security.api-basepath:
    default: v1
    sensitive: false
    required: true
  common.pvm-security.host:
    default: ''
    sensitive: false
    required: true
  common.pvm-security.max-retries:
    default: '2'
    sensitive: false
    required: true
  common.pvm-security.password:
    default: null
    sensitive: true
    required: true
  common.pvm-security.port:
    default: ''
    sensitive: false
    required: true
  common.pvm-security.ssl-port:
    default: ''
    sensitive: false
    required: true
  common.pvm-security.ssl.protocol:
    default: PLAINTEXT
    sensitive: false
    required: true
  common.pvm-security.user:
    default: admin
    sensitive: false
    required: true
  common.security.api-paths.additional-public:
    default: ''
    sensitive: false
    required: true
  common.service.api-basepath:
    default: api
    sensitive: false
    required: true
  common.service.app-basepath:
    default: coordinator
    sensitive: false
    required: true
  common.service.zone-id:
    default: ''
    sensitive: false
    required: true
  management.endpoints.web.exposure.include:
    default: prometheus
    sensitive: false
    required: true
  management.endpoints.web.path-mapping.prometheus:
    default: /metrics
    sensitive: false
    required: true
  server.max-http-header-size:
    default: 32KB
    sensitive: false
    required: true
  server.ssl.client-auth:
    default: WANT
    sensitive: false
    required: true
  server.ssl.enabled:
    default: 'true'
    sensitive: false
    required: true
  server.ssl.key-password:
    default: null
    sensitive: true
    required: true
  server.ssl.key-store:
    default: ''
    sensitive: false
    required: true
  server.ssl.key-store-password:
    default: null
    sensitive: true
    required: true
  server.ssl.trust-store:
    default: ''
    sensitive: false
    required: true
  server.ssl.trust-store-password:
    default: null
    sensitive: true
    required: true
  service.archive.flow.checkpoint-interval-ms:
    default: '600000'
    sensitive: false
    required: true
  service.archive.flow.max-thread-datarate-bytes-per-sec:
    default: '2621440'
    sensitive: false
    required: true
  service.archive.hadoop.filesystem.custom-enabled:
    default: 'true'
    sensitive: false
    required: true
  service.archive.hdfs.storage.query-results-path:
    default: /tengri/archive/results
    sensitive: false
    required: true
  service.archive.hdfs.storage.sources-path:
    default: /tengri/archive/sources
    sensitive: false
    required: true
  service.archive.hdfs.storage.temporary-path:
    default: /tengri/archive/temporary
    sensitive: false
    required: true
  service.archive.overdraft.max-overdraft-percentage:
    default: '50'
    sensitive: false
    required: true
  service.archive.partition.granularity-default-time-unit:
    default: WEEK
    sensitive: false
    required: true
  service.archive.query.result.max-rows:
    default: '-1'
    sensitive: false
    required: true
  service.archive.query.result.max-rows-per-page:
    default: '10000'
    sensitive: false
    required: true
  service.archive.scheduler.partition.merge-enabled:
    default: 'false'
    sensitive: false
    required: true
  service.archive.scheduler.partition.merge-interval-sec:
    default: '86400'
    sensitive: false
    required: true
  service.archive.scheduler.query.result-clean-interval-sec:
    default: '360'
    sensitive: false
    required: true
  service.archive.scheduler.query.result-expired-sec:
    default: '86400'
    sensitive: false
    required: true
  service.archive.scheduler.query.status-update-interval-sec:
    default: '60'
    sensitive: false
    required: true
  service.archive.scheduler.rotation.index-filling-threshold:
    default: '0.7'
    sensitive: false
    required: true
  service.archive.scheduler.rotation.interval-sec:
    default: '360'
    sensitive: false
    required: true
  service.archive.spark.deploy-mode:
    default: cluster
    sensitive: false
    required: true
  service.archive.spark.event-logging-dir:
    default: /tengri/archive/sparkLogs
    sensitive: false
    required: true
  service.archive.spark.event-logging-enabled:
    default: 'false'
    sensitive: false
    required: true
  service.archive.spark.master:
    default: yarn
    sensitive: false
    required: true
  service.archive.yarn.application-submission-timeout-sec:
    default: '60'
    sensitive: false
    required: true
  service.archive.yarn.client.ssl.enabled:
    default: 'false'
    sensitive: false
    required: true
  service.archive.yarn.resource-manager-url:
    default: ''
    sensitive: false
    required: true
  service.jvm-params:
    default: -Xmx1g -XX:+ExitOnOutOfMemoryError -XX:MaxGCPauseMillis=100 -XX:+UseG1GC
    sensitive: false
    required: true
  service.pid-directory:
    default: /var/run/abyss
    sensitive: false
    required: true
  service.port:
    default: '29103'
    sensitive: false
    required: true
  service_properties:
    default: null
    sensitive: false
    required: true
  spring.datasource.hikari.connection-timeout:
    default: '30000'
    sensitive: false
    required: true
  spring.datasource.hikari.idle-timeout:
    default: '300000'
    sensitive: false
    required: true
  spring.datasource.hikari.leak-detection-threshold:
    default: '0'
    sensitive: false
    required: true
  spring.datasource.hikari.max-lifetime:
    default: '1200000'
    sensitive: false
    required: true
  spring.datasource.hikari.maximum-pool-size:
    default: '6'
    sensitive: false
    required: true
  spring.datasource.hikari.minimum-idle:
    default: '6'
    sensitive: false
    required: true
  spring.datasource.password:
    default: null
    sensitive: true
    required: true
  spring.datasource.type:
    default: com.zaxxer.hikari.HikariDataSource
    sensitive: false
    required: true
  spring.datasource.url:
    default: ''
    sensitive: false
    required: true
  spring.datasource.username:
    default: archive_service_user
    sensitive: false
    required: true
  spring.flyway.password:
    default: null
    sensitive: true
    required: true
  spring.flyway.user:
    default: flyway_archive_service_user
    sensitive: false
    required: true
archive-logback-config:
  logback_properties:
    default: null
    sensitive: false
    required: true
  logger.directory:
    default: /var/log/abyss
    sensitive: false
    required: true
  logger.file-size:
    default: 100MB
    sensitive: false
    required: true
  logger.kafka.bootstrap-servers:
    default: ''
    sensitive: false
    required: true
  logger.kafka.client-id:
    default: abyss_archive_service
    sensitive: false
    required: true
  logger.kafka.protocol:
    default: PLAINTEXT
    sensitive: false
    required: true
  logger.kafka.topic:
    default: abyss.coordinator_logs_ambari
    sensitive: false
    required: true
  logger.max-file-amount:
    default: '10'
    sensitive: false
    required: true

service_id: service_name
service_type: SERVICE_TYPE
default_config_group
  logic_config_group_1:
    key: value
	  ...
  logic_config_group_N:
    key: value
	  ...
custom_config_group:
  logic_config_group_N
    key: value
	  ...

service_id: service_name

service_type: SERVICE_TYPE

default_config_group
  logic_config_group_1:
    key: value
	  ...
  logic_config_group_N:
    key: value
	  ...

service_id: Abyss_Auth_service
service_type: AUTH_SERVICE
default_config_group:
  auth-logback-config:
    logger.file-size: '{{ logger_file_size }}'
    logger.kafka.bootstrap-servers: '{{ logger_kafka_bootstrap_servers }}'
    logger.kafka.protocol: '{{ logger_kafka_protocol }}'
    logger.kafka.topic: '{{ logger_kafka_topic }}'
    logger.max-file-amount: '{{ logger_max_file_amount }}'
  auth-service-config:
    abyss.audit.event.publish-url: '{{ abyss_audit_event_publish_url }}'
    common.actuator.basic-auth.enabled: '{{ actuator_basic_auth_enabled }}'
    common.actuator.basic-auth.user: '{{ actuator_basic_auth_user }}'
    common.actuator.basic-auth.password: '{{ actuator_basic_auth_password }}'
    common.auth.jwk-resource-server: '{{ auth_jwk_resource_server }}'
    common.auth.secret: '{{ jwt_secret }}'
    common.discovery.zookeeper.connection: '{{ zk_hosts }}'
    common.discovery.zookeeper.path: '{{ services_zk_chroot }}'
    common.global.client.ssl.key-password: '{{ ssl.key_password }}'
    common.global.client.ssl.keystore-location: '{{ ssl_keystore_location }}'
    common.global.client.ssl.keystore-password: '{{ ssl.keystore_password }}'
    common.global.client.ssl.truststore-location: '{{ ssl_truststore_location }}'
    common.global.client.ssl.truststore-password: '{{ ssl.truststore_password }}'
    common.internal-communication.data-center-id: '{{ common_internal_communication_data_center_id }}'
    common.internal-communication.ssl.protocol: '{{ common_internal_communication_ssl_protocol }}'
    common.pvm-security.host: '{{ pvm_security_host }}'
    common.pvm-security.password: '{{ pvm_internal_user_password }}'
    common.pvm-security.port: '{{ pvm_security_port }}'
    common.pvm-security.ssl.protocol: '{{ pvm_security_ssl_protocol }}'
    common.pvm-security.ssl-port: '{{ pvm_security_ssl_port }}'
    common.pvm-security.user: '{{ pvm_internal_user_name }}'
    service.auth.ldap.password: '{{ ldap_user_password }}'
    service.auth.ldap.search-dns: '["DC=domen,DC=sbt,DC=ru"]'
    service.auth.ldap.url: ldap://
    service.auth.ldap.username: ldap_user@domen.ru
    service.auth.admin.username: '{{ pvm_internal_user_name }}'
    service.auth.admin.password: '{{ pvm_internal_user_password }}'
    service.jvm-params: -Xmx64m -XX:+ExitOnOutOfMemoryError -XX:MaxGCPauseMillis=100 -XX:+UseG1GC
    server.ssl.enabled: '{{ service_ssl_enable }}'
    server.ssl.key-password: '{{ ssl.key_password }}'
    server.ssl.key-store: '{{ ssl_keystore_location }}'
    server.ssl.key-store-password: '{{ ssl.keystore_password }}'
    server.ssl.trust-store: '{{ ssl_truststore_location }}'
    server.ssl.trust-store-password: '{{ ssl.truststore_password }}'
    spring.datasource.url: jdbc:postgresql://{{ database_host }}:{{ database_port }}/{{ global_database_name }}
    spring.datasource.password: '{{ global_database_password }}'
    spring.flyway.password: '{{ flyway_password }}'

services:
  service_1_id:
    config_groups:
      default_config_group:
        hosts_group:
        - service_id_host_1
      custom_config_group:
        hosts_group:
        - service_id_host_2
    roles:
      role_1:
        hosts_group:
        - service_id_host_1
        - service_id_host_2
  ...

group_host:
  hosts:
    - host1
    - host2
...

host1: ip1
host2: ip2

services:
  ambari_server:
    hosts_group:
    - ambari_server_hosts
  Abyss_Archive_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      archive:
        hosts_group:
        - ambari_server_hosts
  Abyss_Auth_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      auth:
        hosts_group:
        - ambari_server_hosts
  PVM_AUTHORIZATION_SERVICE:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      pvm_authorization:
        hosts_group:
        - ambari_server_hosts
  Abyss_Client_Api_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      client_api:
        hosts_group:
        - ambari_server_hosts
  Abyss_Flow_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      flow:
        hosts_group:
        - ambari_server_hosts
  Abyss_FullTextIndex_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      fulltextindex:
        hosts_group:
        - ambari_server_hosts
  Abyss_Group_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      group:
        hosts_group:
        - ambari_server_hosts
  Abyss_Kafka_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      kafka:
        hosts_group:
        - ambari_server_hosts
  Abyss_Monitoring_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      monitoring:
        hosts_group:
        - ambari_server_hosts
  Abyss_Pipeline_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      pipeline:
        hosts_group:
        - ambari_server_hosts
  Abyss_Project_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      project:
        hosts_group:
        - ambari_server_hosts
  Abyss_Role_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      role:
        hosts_group:
        - ambari_server_hosts
  TraceCollector_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      trace_collector_server:
        hosts_group:
        - ambari_server_hosts
  TraceQuery_service:
    config_groups:
      default_config_group:
        hosts_group:
        - ambari_server_hosts
    roles:
      trace_query:
        hosts_group:
        - ambari_server_hosts
  ABYSS_SOLR:
      config_groups:
        default_config_group:
          hosts_group:
          - solr_hosts
      roles:
        abyss_solr_server:
          hosts_group:
          - solr_hosts

nginx_hosts:
  hosts:
    - abyss-install-5-postgres
coordinator_hosts:
  hosts:
    - abyss-install-2
yarn_jobhistory_hosts:
  hosts:
    - abyss-install-3
yarn_rm_hosts:
  hosts:
    - abyss-install-2
    - abyss-install-3

zookeeper_hosts:
  hosts:
    - abyss-install-1
    - abyss-install-2
    - abyss-install-3
hdfs_namenodes_hosts:
  hosts:
    - abyss-install-1
    - abyss-install-2
hdfs_datanodes_hosts:
  hosts:
    - abyss-install-2
    - abyss-install-3
    - abyss-install-4
hdfs_balancer_hosts:
  hosts:
    - abyss-install-4
hdfs_httpfs_hosts:
  hosts:
    - abyss-install-2
hdfs_journalnode_hosts:
  hosts:
    - abyss-install-2
    - abyss-install-3
    - abyss-install-4

abyss-install-1: ip
abyss-install-2: ip
abyss-install-3: ip
abyss-install-4: ip
abyss-install-5-postgres: ip

preset_name: "v1.0"

ssl_truststore_location: "/abyss/ssl/server.truststore.jks"
zookeeper_hosts: "host1:port"

service_id: service_name
service_type: SERVICE_TYPE
default_config_group:
  auth-service-config:
    abyss.audit.event.publish-url:
      ssl.keystore_location: "{{ ssl_truststore_location }}"
      ...

ansible_user: "my_ansible_user"
ansible_become: "false"

abyss_distr_dir: "/abyss/install"

coordinator_ui_path: "/abyss/tengri-ui"

  ansible-vault encrypt --vault-password-file <path_to_vault_password_file> pass.yml

  ansible-vault decrypt --vault-password-file <path_to_vault_password_file> pass.yml

cluster_manager_password: "pass"
global_database_password: "pass"

ssl:
  key_password: "pass"
  keystore_password: "pass"
  truststore_password: "pass"

service_id: service_name
service_type: SERVICE_TYPE
default_config_group:
  auth-service-config:
    logger_ssl_client_truststore_location: "{{ ssl_truststore_location }}"
    logger_ssl_client_truststore_password: "{{ ssl.truststore_password }}"
    logger_ssl_server_keystore_location: /abyss/ssl/server.keystore.jks
    logger_ssl_server_keystore_password: "{{ ssl.keystore_password }}"

preset_name: "v1.0"

zk_hosts=abyss-install-1:2181,abyss-install-2:2181,abyss-install-3:2181
kafka_bootstrap_servers=abyss-install-4:9093,abyss-install-3:9093,abyss-install-2:9093
druid_router_host=abyss-install-5-postgres

yarn_rm_host=abyss-install-3
flink_masters_hosts=abyss-install-1,abyss-install-2
database_host=abyss-install-5-postgres
database_port=5432


tracing_url=http://host:port/api/v2/spans
ssl_truststore_location=/tengri/ssl/server.truststore.jks
ssl_keystore_location=/tengri/ssl/server.keystore.jks

ssl_protocol=SSL_WANT
kafka_security_protocol=SSL

pvm_internal_user_name=admin


solr_basic_user=admin
solr_basic_enabled=true

# Dynamic params
hue_webhdfs_name='{ "SERVICE_TYPE":"HDFS","SERVICE_ROLE":"HDFS-HTTPFS","VALUE_PARAM":"name" }'

ansible_user: "root"

# user, password, port and scheme for Ambari
cluster_manager_user: "admin"
cluster_manager_password: "{{cluster_manager_password}}"
cluster_manager_port: "8080"
cluster_manager_scheme: "http"

abyss_distr_dir: "/tengri/install"

ssl_deploy_dir: "/tengri/ssl"

cluster_manager_password: "pass"

global_database_password: "pass"

ssl:
  key_password: "pass"
  keystore_password: "pass"
  truststore_password: "pass"

jwt_secret: "secret"
jwt_encodng_key: "pass"

pvm_internal_user_password: "pass"

solr_basic_password: "pass"

druid_basic_password: "pass"