![]() option("startingPosition", "TRIM_HORIZON") ![]() option("streamName", "stream-join-demo") Val datasource0 = sparkSession.readStream // readstream() returns type DataStreamReader load("s3://awsexamplebucket-streaming-demo2/inputs/productsStatic.csv") // load() returns a DataFrame Val staticData = sparkSession.read // read() returns type DataFrameReader Job.init(args("JOB_NAME"), glueContext, args.asJava) Val args = GlueArgParser.getResolvedOptions(sysArgs, Seq("JOB_NAME").toArray) Val sparkSession: SparkSession = glueContext.getSparkSession Val glueContext: GlueContext = new GlueContext(spark) ![]() Val spark: SparkContext = new SparkContext() joins the stream to a static dataset on Amazon S3, and outputs the joined results to Amazon S3 in parquet format. This script connects to an Amazon Kinesis stream, uses a schema from the data catalog to parse the stream,
0 Comments
Leave a Reply. |