Spark

parser

This example show how to add date argument into dataframe

import argparsefrom pyspark.sql import SparkSession
parser = argparse.ArgumentParser() parser.add_argument('--d', required=True, dest='test')args = parser.parse_args()
date_filter = f"(date_partition >= '{args.test}')"
spark = SparkSession.builder.getOrCreate()
df = spark.sql(f"""select id,date_partitionfrom orc.`/datalake/system/customers/`where {date_filter}""")