25 lines
736 B
Python
25 lines
736 B
Python
from clearml import PipelineDecorator
|
|
|
|
@PipelineDecorator.component(cache=True, execution_queue="default")
|
|
def step(size: int):
|
|
import numpy as np
|
|
return np.random.random(size=size)
|
|
|
|
@PipelineDecorator.pipeline(
|
|
name='ingest',
|
|
project='data processing',
|
|
version='0.1'
|
|
)
|
|
def pipeline_logic(do_stuff: bool):
|
|
if do_stuff:
|
|
return step(size=42)
|
|
|
|
if __name__ == '__main__':
|
|
# run the pipeline on the current machine, for local debugging
|
|
# for scale-out, comment-out the following line (Make sure a
|
|
# 'services' queue is available and serviced by a ClearML agent
|
|
# running either in services mode or through K8S/Autoscaler)
|
|
PipelineDecorator.run_locally()
|
|
|
|
pipeline_logic(do_stuff=True)
|