ENV VARIABLES:

"SPARK_HOME": "C:\\Spark\\spark-3.5.1-bin-hadoop3"
"PYSPARK_PYTHON": "C:\\Users\\User\\anaconda3\\python.exe"
"PYSPARK_DRIVER_PYTHON": "C:\\Users\\User\\anaconda3\\python.exe"

C:\>pyspark
Python 3.11.7 | packaged by Anaconda, Inc. | (main, Dec 15 2023, 18:05:47) [MSC v.1916 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license" for more information.
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /__ / .__/\_,_/_/ /_/\_\   version 3.5.1
      /_/

Using Python version 3.11.7 (main, Dec 15 2023 18:05:47)
Spark context Web UI available at http://10.0.0.202:4040
Spark context available as 'sc' (master = local[*], app id = local-1715375102664).
SparkSession available as 'spark'.
>>> book_rdd = sc.textFile("C:/Users/User/OneDrive/Desktop/BigData/Assignments/205-0.txt")
>>> book_rdd.take(10)
['The Project Gutenberg eBook of Walden, by Henry David Thoreau', '', 'This eBook is for the use of anyone anywhere in the United States and', 'most other parts of the world at no cost and with almost no restrictions', 'whatsoever. You may copy it, give it away or re-use it under the terms', 'of the Project Gutenberg License included with this eBook or online at', 'www.gutenberg.org. If you are not located in the United States, you', 'will have to check the laws of the country where you are located before', 'using this eBook.', '']
>>>
>>> def clean_word(word):
...     if word[-1] in '.,;?:':
...         word = word[:-1]
...     if word.isalpha() and len(word) >= 5:
...         return word.lower()
...     return None
...
>>> cleaned_words_rdd = book_rdd.flatMap(lambda line: line.split().map(clean_word).filter(lambda word: word is not None)

...
... )
>>> cleaned_words_rdd = book_rdd.flatMap(lambda line: line.split()).map(clean_word).filter(lambda word: word is not None)
>>> word_count_rdd = cleaned_words_rdd.map(lambda word: (word, 1)).reduceByKey(lambda a, b: a+b)
>>> sorted_word_count_rdd = word_count_rdd.sortBy(lambda pair: pair[1], ascending = False)
>>> top_words = sorted_word_count_rdd.take(10)
>>> for word, count in top_words:
...     print(f"({word}, {count})")
...
(which, 870)
(their, 503)
(there, 394)
(would, 337)
(where, 220)
(never, 188)
(should, 184)
(could, 182)
(house, 182)
(other, 181)