Skip to content

Exporting

clean_files(upload)

Cleans the files generated by the pipeline in data directory.

Parameters:

Name Type Description Default
upload Dict[str, bool]

Dictionary of outputs from the upload step.

 required

Returns:

Type Description
Dict[str, bool]

Dictionary of outputs from the clean step.
Source code in make_us_rich/pipelines/exporting/nodes.py 
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def clean_files(upload: Dict[str, bool]) -> Dict[str, bool]:
    """
    Cleans the files generated by the pipeline in data directory.

    Parameters
    ----------
    upload: Dict[str, bool]
        Dictionary of outputs from the upload step.

    Returns
    -------
    Dict[str, bool]
        Dictionary of outputs from the clean step.
    """
    if upload["upload_done"] == True:
        data_folders = [folder[0] for folder in os.walk("data") if folder[0] != "data"]
        for folder in data_folders:
            files = os.listdir(folder)
            for file in files:
                os.remove(f"{folder}/{file}") if file != ".gitkeep" else None
        return {"clean_done": True}
    return {"clean_done": False}

upload_files(currency, compare, dir_path, validation)

Uploads model and features engineering files to Minio.

Parameters:

Name Type Description Default
currency str

Currency used in the model.

 required
compare str

Compare used in the model.

 required
validation Dict[Tuple[str, bool], Tuple[str, str]]

Dictionary of outputs from the validation step.

 required
dir_path str

Directory path where the model files are saved.

 required

Returns:

Type Description
Dict[Tuple[str, bool]]

Dictionary of outputs from the upload step.
Source code in make_us_rich/pipelines/exporting/nodes.py 
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
def upload_files(
    currency: str,
    compare: str,
    dir_path: str,
    validation: Dict[Tuple[str, bool], Tuple[str, str]],
) -> Dict[str, bool]:
    """
    Uploads model and features engineering files to Minio.

    Parameters
    ----------
    currency: str
        Currency used in the model.
    compare: str
        Compare used in the model.
    validation: Dict[Tuple[str, bool], Tuple[str, str]]
        Dictionary of outputs from the validation step.
    dir_path: str
        Directory path where the model files are saved.

    Returns
    -------
    Dict[Tuple[str, bool]]
        Dictionary of outputs from the upload step.
    """
    if validation["validation_done"] == True:
        client = MinioClient()
        date = datetime.now().strftime("%Y-%m-%d")
        model_path = f"{dir_path}/model.onnx"
        client.upload(client.bucket, f"{date}/{currency}_{compare}/model.onnx", model_path)
        scaler_path = f"{dir_path}/scaler.pkl"
        client.upload(client.bucket, f"{date}/{currency}_{compare}/scaler.pkl", scaler_path)
        return {"upload_done": True}
    return {"upload_done": False}
Last update: 2022-05-04