diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..1ce816f --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,22 @@ +name: Java CI with Maven + +on: + push: + branches: [ "main", "develop" ] + pull_request: + branches: [ "main", "develop" ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + - name: Build and test with Maven + run: mvn -B test --file pom.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e2b8312 --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + +# IntelliJ IDEA +.idea/ +*.iml +*.iws +out/ + +# Eclipse +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# VS Code +.vscode/ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db diff --git a/Help/SortedByTitle.txt b/Help/SortedByTitle.txt new file mode 100644 index 0000000..585052a --- /dev/null +++ b/Help/SortedByTitle.txt @@ -0,0 +1,15 @@ +1. Movie: 12 Angry Men, Rating: 9.0, Year: 1957 +2. Movie: Fight Club, Rating: 8.8, Year: 1999 +3. Movie: Forrest Gump, Rating: 8.8, Year: 1994 +4. Movie: Goodfellas, Rating: 8.7, Year: 1990 +5. Movie: Inception, Rating: 8.8, Year: 2010 +6. Movie: Interstellar, Rating: 8.6, Year: 2014 +7. Movie: Pulp Fiction, Rating: 8.9, Year: 1994 +8. Movie: Schindler's List, Rating: 9.0, Year: 1993 +9. Movie: Star Wars: Episode V - The Empire Strikes Back, Rating: 8.7, Year: 1980 +10. Movie: The Dark Knight, Rating: 9.0, Year: 2008 +11. Movie: The Godfather, Rating: 9.2, Year: 1972 +12. Movie: The Lord of the Rings: The Fellowship of the Ring, Rating: 8.8, Year: 2001 +13. Movie: The Lord of the Rings: The Return of the King, Rating: 9.0, Year: 2003 +14. Movie: The Matrix, Rating: 8.7, Year: 1999 +15. Movie: The Shawshank Redemption, Rating: 9.3, Year: 1994 \ No newline at end of file diff --git a/Help/SortedByYear.txt b/Help/SortedByYear.txt new file mode 100644 index 0000000..3e7488c --- /dev/null +++ b/Help/SortedByYear.txt @@ -0,0 +1,15 @@ +1. Movie: Interstellar, Rating: 8.6, Year: 2014 +2. Movie: Inception, Rating: 8.8, Year: 2010 +3. Movie: The Dark Knight, Rating: 9.0, Year: 2008 +4. Movie: The Lord of the Rings: The Return of the King, Rating: 9.0, Year: 2003 +5. Movie: The Lord of the Rings: The Fellowship of the Ring, Rating: 8.8, Year: 2001 +6. Movie: Fight Club, Rating: 8.8, Year: 1999 +7. Movie: The Matrix, Rating: 8.7, Year: 1999 +8. Movie: The Shawshank Redemption, Rating: 9.3, Year: 1994 +9. Movie: Pulp Fiction, Rating: 8.9, Year: 1994 +10. Movie: Forrest Gump, Rating: 8.8, Year: 1994 +11. Movie: Schindler's List, Rating: 9.0, Year: 1993 +12. Movie: Goodfellas, Rating: 8.7, Year: 1990 +13. Movie: Star Wars: Episode V - The Empire Strikes Back, Rating: 8.7, Year: 1980 +14. Movie: The Godfather, Rating: 9.2, Year: 1972 +15. Movie: 12 Angry Men, Rating: 9.0, Year: 1957 \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8a03ac6 --- /dev/null +++ b/README.md @@ -0,0 +1,146 @@ +# Fourth Assignment: IMDb Scraper 🎬 + +## Introduction + +This project delves into the practical application of HTML parsing techniques using Java and the JSoup library. The objective is to extract and manipulate data from a structured HTML document containing information about various movies. Key tasks include parsing the document, creating Java objects to represent the data, sorting the information based on different criteria, and presenting the results effectively. + +## Why Learn Web Scraping? 📊 + +Mastering web scraping empowers individuals and businesses to glean valuable insights, automate tasks, and make informed decisions based on real-time data. + +- In e-commerce, web scraping aids in monitoring competitors' prices, tracking product availability, and gathering customer reviews. In finance, it facilitates trend analysis, investment decisions, and risk assessment. + +- Research, academia, journalism, and media professionals leverage web scraping for data collection, trend tracking, sentiment analysis, and investigative reporting. + +By mastering web scraping, professionals can extract data efficiently, outperform competitors, and drive data-informed success in diverse industries. + +## Prerequisites ✅ + +Before diving into this project, make sure you have the following tools and knowledge: + +### Tools Needed: +- **Java 17 or higher** : Download the latest version from [Oracle's Java website](https://www.oracle.com/java/technologies/downloads/). +- **Git** : Install Git, a powerful version control system, from [Git's official website](https://git-scm.com/downloads). +- **Maven** : Utilize Maven as the build tool. You can find installation instructions [here](https://maven.apache.org/install.html). + +### Knowledge Required: +In addition to the tools mentioned above, familiarity with the following topics is crucial for successful project completion: +- **Jsoup** : Understanding how to access, parse, and manipulate HTML files using Jsoup. +- **HTML** : Knowledge of various HTML tags, their purposes, and how to identify and manipulate them using Jsoup. +- **CSS** : Understanding how to style HTML elements, work with CSS classes, and utilize selectors efficiently. + +By having a solid grasp of these tools and concepts, you'll be well-equipped to excel in this project. 🚀 + +## Objectives 🎯 + +The primary objectives of this assignment are: + +- Parsing an HTML file with movie data. +- Extracting relevant information such as title, rating, and release year. +- Implementing Java classes to represent movies. +- Utilizing JSoup for HTML parsing. +- Applying sorting algorithms for organizing data by title, release year, and rating. +- Displaying the sorted movie information effectively. +- Passing the provided **JUnit tests** to verify the correctness of your code. + +## Tasks 📝 + +1. **Set Up the Project:** + + - Start by cloning the repository to your local machine. + - Open the project in an IDE such as IntelliJ. + +2. **Integrate JSoup Dependency:** + + - Include the JSoup dependency in your `pom.xml` file. + + ```xml + + org.jsoup + jsoup + 1.14.3 + + ``` + +3. **Complete the Movie Class:** + + - Complete the `Movie` class in Java to encapsulate movie-specific data. + +4. **HTML Parsing using JSoup:** + + Utilize JSoup to parse the HTML file located in the resources folder at this path: `src/main/resources/Movies.html` and extract relevant information such as: + + - `` = for movie titles + - `` = for movie ratings + - `` = for movie release years + + **Note: The span tags are nested within other tags and div closures. Look out for unneeded text like '/10' inside elements.** + +5. **Implement Sorting Functionality:** + + - Develop sorting algorithms for various criteria like title, year, and rating. + +6. **Run JUnit Tests:** + + - We have provided tests using JUnit 5 (`ParserTest.java`). Run them via your IDE or by executing the following Maven command in your terminal to verify your logic: + ```bash + mvn test + ``` + +7. **Visualize Results:** + + - Clearly present the sorted movie data. + +## Bonus Objectives (For Advanced Users) 🌟 + +To enhance your project further: + +- Expand data extraction to additional websites, such as [Oscar Winning Films](https://www.scrapethissite.com/pages/ajax-javascript/). +- Develop a user interface for interactive querying and sorting. +- Utilize SQL or NoSQL databases to store the scraped data. + +## Notes 📝 + +Here are some important points to keep in mind: + +- There is a `Help` folder located at the root directory of the project which contains the sorted movies each by different priorities. It is there just so you can see what your output should look like in the end. +- The unit tests are provided to assist you in understanding the project requirements. Your final grade is not solely dependent on their results; they are meant to aid your learning process. Remember to enable GitHub Actions for the test workflow to run on GitHub. +- Feel free to leverage ChatGPT for learning web scraping and resolving any challenges that you may not find solutions to on the internet. Utilize its capabilities to enhance your understanding and overcome obstacles effectively. +**But it is strictly prohibited to use ChatGPT or any other AI generative model for completing any section of this assignment. Failure to comply will result in a score of 0 without any warnings.** + +## Evaluation 🧐 + +Your success will be judged based on: + +- Functionality of the code in parsing HTML, creating Java objects, and sorting data. +- Readability and maintainability of the code. +- Efficiency of sorting algorithms. +- Clarity of the output format. +- Successful execution of the provided JUnit tests. + +## Submission 📁 + +To share your work: + +1. Add your mentor as a contributor to the project. +2. Create a `develop` branch for implementing features. +3. Use Git for regular code commits. +4. Push your code to the remote repository. +5. Submit a pull request to merge the `develop` branch with `main`. + +If you have any further questions or need clarification, do not hesitate to reach out to your mentor. Good luck with your project! 🌟 + +## TA Tutorial Videos 🎥 + +To help you succeed in this assignment, our TA team has recorded dedicated tutorial videos covering the essential topics. Please watch these before starting: + +- [Tutorial: Basics of HTML & CSS]([INSERT_LINK_HERE]) +- [Tutorial: Web Scraping in Java using JSoup]([INSERT_LINK_HERE]) + +## Additional Resources 📚 + +- [Jsoup - Official website](https://jsoup.org/cookbook/) +- [Jsoup - Javapoint](https://www.javatpoint.com/jsoup-tutorial) +- [Jsoup - Baeldung](https://www.baeldung.com/java-with-jsoup) +- [HTML - W3](https://www.w3schools.com/html/) +- [CSS - W3](https://www.w3schools.com/css/default.asp) diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..c187af2 --- /dev/null +++ b/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + + org.example + HTML-Parser + 1.0-SNAPSHOT + + + 17 + 17 + + + + + org.junit.jupiter + junit-jupiter-api + 5.8.1 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.8.1 + test + + + org.jsoup + jsoup + 1.14.3 + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + + \ No newline at end of file diff --git a/src/main/java/Movie.java b/src/main/java/Movie.java new file mode 100644 index 0000000..fcdcf30 --- /dev/null +++ b/src/main/java/Movie.java @@ -0,0 +1,37 @@ +import java.util.Objects; + +public class Movie { + private String title; + private double rating; + private int year; + + public Movie(String title, double rating, int year) { + //TODO + } + + public String getTitle() { + return title; + } + + public double getRating() { + return rating; + } + + public int getYear() { + return year; + } + + @Override + public String toString() { + //TODO + return ""; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Movie movie = (Movie) o; + return Double.compare(movie.rating, rating) == 0 && movie.year == year && Objects.equals(title, movie.title); + } +} \ No newline at end of file diff --git a/src/main/java/Parser.java b/src/main/java/Parser.java new file mode 100644 index 0000000..f69045b --- /dev/null +++ b/src/main/java/Parser.java @@ -0,0 +1,47 @@ +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import java.io.File; +import java.io.IOException; +import java.util.*; + +public class Parser { + static List movies = new ArrayList<>(); + + public List sortByTitle() { + List sortedByTitle = new ArrayList<>(movies); + // Sort movies alphabetically by title + //TODO + return sortedByTitle; + } + + public List sortByRating() { + List sortedByRating = new ArrayList<>(movies); + // Sort movies by rating (highest to lowest) + //TODO + return sortedByRating; + } + + public List sortByYear() { + List sortedByYear = new ArrayList<>(movies); + // Sort movies by year (newest to oldest) + //TODO + return sortedByYear; + } + + public void setUp() throws IOException { + // Parse the HTML file using Jsoup + //TODO + + // Extract data from the HTML + //TODO + + // Iterate through each movie div to extract movie data + //TODO + } + + public static void main(String[] args) { + // You can test your code here before you run the unit tests + } +} \ No newline at end of file diff --git a/src/main/resources/Movies.html b/src/main/resources/Movies.html new file mode 100644 index 0000000..b2ccd76 --- /dev/null +++ b/src/main/resources/Movies.html @@ -0,0 +1,119 @@ +
+
+
+
+
+

Movie List: To Scrape

+
+
+
+
+
+

The Godfather

+
+ Rating: 9.2/10
+ Year: 1972
+
+
+
+

The Shawshank Redemption

+
+ Rating: 9.3/10
+ Year: 1994
+
+
+
+

The Dark Knight

+
+ Rating: 9.0/10
+ Year: 2008
+
+
+
+

12 Angry Men

+
+ Rating: 9.0/10
+ Year: 1957
+
+
+
+

Schindler's List

+
+ Rating: 9.0/10
+ Year: 1993
+
+
+
+

The Lord of the Rings: The Return of the King

+
+ Rating: 9.0/10
+ Year: 2003
+
+
+
+

Pulp Fiction

+
+ Rating: 8.9/10
+ Year: 1994
+
+
+
+

The Lord of the Rings: The Fellowship of the Ring

+
+ Rating: 8.8/10
+ Year: 2001
+
+
+
+

Forrest Gump

+
+ Rating: 8.8/10
+ Year: 1994
+
+
+
+

Fight Club

+
+ Rating: 8.8/10
+ Year: 1999
+
+
+
+

Inception

+
+ Rating: 8.8/10
+ Year: 2010
+
+
+
+

Star Wars: Episode V - The Empire Strikes Back

+
+ Rating: 8.7/10
+ Year: 1980
+
+
+
+

The Matrix

+
+ Rating: 8.7/10
+ Year: 1999
+
+
+
+

Goodfellas

+
+ Rating: 8.7/10
+ Year: 1990
+
+
+
+

Interstellar

+
+ Rating: 8.6/10
+ Year: 2014
+
+
+
+
+
+
\ No newline at end of file