veld_code__nlp_generic_preprocessing/veld_preprocess_sample.yaml at main · veldhub/veld_code__nlp_generic_preprocessing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# TODO: implement
x-veld:
  code:
    description: "takes a random sample of lines from a txt file. Randomness can be set with a seed
      too"
    topic:
      - "NLP"
      - "Preprocessing"
      - "ETL"

    input:
      - volume: /veld/input/
        environment_var: in_txt_file
        file_type: "txt"
        content: "raw text"

    output:
      - volume: /veld/output/
        environment_var: out_txt_file
        file_type: "txt"
        content: "raw text"

    config:
      - environment_var: out_data_description
        description: "automatic data description for generating a data veld yaml file"
        var_type: "str"
      - environment_var: percentage_sample
        description: "percentage of lines to be randomly sampled"
        var_type: "int"
      - environment_var: sample_random_seed
        description: "seed to make randomness stable and reproducible"
        var_type: "str"
      - environment_var: buffer_segments
        description: "percentage of segments where processing results are persisted in between. So
          that processing could continue should it have crashed"
        var_type: "int"
        default: 100

services:
  veld_preprocess_sample:
    build: .
    volumes:
      - ./src/:/veld/code/:z
    command: python -u /veld/code/preprocess.py sample
    environment:
      in_file: null
      out_file: null
      out_data_description: null
      percentage_sample: null
      absolute_sample: null
      sample_random_seed: null
      buffer_size: 1