Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /__ / .__/\_,_/_/ /_/\_\   version 3.5.1
      /_/

Using Python version 3.12.3 (main, Apr  9 2024 16:03:47)
Spark context Web UI available at http://10.0.0.248:4040
Spark context available as 'sc' (master = local[*], app id = local-1715654853973).
SparkSession available as 'spark'.
>>> nums = [1, -1, -5, 2, 3, 6, 7, 8, 9, 10]
>>> nums_rdd = sc.parallelize(nums)
>>> nums.collect()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: 'list' object has no attribute 'collect'
>>> rdd_nums.collect()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
NameError: name 'rdd_nums' is not defined
>>> nums_rdd.collect()
[1, -1, -5, 2, 3, 6, 7, 8, 9, 10]
>>> 
>>> # Kep only the positive elements
>>> positives = nums_rdd.filter(lambda x: x > 0)
>>> positives.collect()
[1, 2, 3, 6, 7, 8, 9, 10]
>>> 
>>> # Calculate the total of all positive elements
>>> total_positives = positives.reduce(lambda x, y: x + y)
>>> total_positives
46
>>> # Calculate the average
>>> # Method 1: Directly using the mean function from PySpark
>>> avg_pos = positives.mean()
>>> avg_pos
5.75
>>> 
>>> # Method 2: Writing meean function from scratch
>>> pairs = positives.map(lambda x: (x,1))
>>> pairs.collect()
[(1, 1), (2, 1), (3, 1), (6, 1), (7, 1), (8, 1), (9, 1), (10, 1)]
>>> sum_count = pairs.reduce(lambda x, y: (x[0]+y[0], x[1]+y[1]))
>>> sum_count
(46, 8)
>>> avg = sum_count[0]+sum_count[1] / 2
>>> avg
50.0
>>> avg = (sum_count[0]+sum_count[1]) / 2
>>> avg
27.0
>>> corrected_avg = (sum_count[0] + sum_count[1]) / 2
>>> corrected_avg
27.0
>>> sum_count
(46, 8)
>>> corrected_avg = sum_count[0] / sum_count[1]
>>> corrected_avg
5.75
>>> 
>>> 
>>> 
>>> nums2 = [2, 3, 4, 5, 6]
>>> rdd2 = sc.parallelize(nums2)
>>> rdd2.collect()
[2, 3, 4, 5, 6]
>>> 
>>> # Add each value by 10
>>> addBy10 = rdd2.mapValues(lambda x: x + 10)

>>> nums3 = [('a', 2), ('a', 3), ('b', 4), ('b', 5), ('b', 6), ('b', 7)]
>>> tup = sc.parallelize(nums3)
>>> tup.collect()
[('a', 2), ('a', 3), ('b', 4), ('b', 5), ('b', 6), ('b', 7)]
>>> 
>>> #---------------------------
>>> addBy10 = rdd2.map(lambda x: x + 10)
>>> addBy10
PythonRDD[9] at RDD at PythonRDD.scala:53
>>> addBy10.collect()
[12, 13, 14, 15, 16]
>>> #---------------------------
>>> 
>>> add10 = tup.mapValues(lambda x: x + 10)
>>> add10.collect()
[('a', 12), ('a', 13), ('b', 14), ('b', 15), ('b', 16), ('b', 17)]
>>> 
>>> 
>>> 
>>> # find min using groupByKey()
>>> min_per_key = tup.groupByKey().mapValues(lambda x: min(x))
>>> min_per_key.collect()
[('a', 2), ('b', 4)]
>>> 
>>> # Now find min, max per key
>>> min_max_per_key = tup.groupByKey().mapValues(lambda x: (min(x), max(x)))
>>> min_max_per_key.collect()
[('a', (2, 3)), ('b', (4, 7))]
>>> grouped = tup.groupByKey()
>>> grouped.collect()
[('a', <pyspark.resultiterable.ResultIterable object at 0x105175820>), ('b', <pyspark.resultiterable.ResultIterable object at 0x105175130>)]
>>> grouped.mapValues(lambda x: list(x))
PythonRDD[26] at RDD at PythonRDD.scala:53
>>> print(grouped.mapValues(lambda x: list(x)))
PythonRDD[27] at RDD at PythonRDD.scala:53
>>> reduced = tup.reduceByKey(lambda x, y: min(x, y))
>>> reduced.collect()
[('a', 2), ('b', 4)]