[JSOUP] Hiscores Lookup Snippet

Chris · September 23, 2018


import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.*;

public class HiScores {

    public static HashMap<String, Stats> find(String searchPhrase){
        HashMap<String, Stats> statsHashMap = new HashMap<>();

        try {
            //Jsoup closes the connection by its own, after the request is done
            Document document = Jsoup
                    .connect("https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws?user1=" + searchPhrase.replaceAll(" ", "%20"))
                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")
                    .timeout(3000)
                    .get();
            //Select the first table.
            Element table = document.select("table").get(0);
            //Grab our rows of td from Element table
            Elements rows = table.select("td");
            //Iterator to remove unnecessary data
            Iterator<Element> elementIterator = rows.iterator();
            //List holds our skill type names
            List<String> typesAvailable = new ArrayList<>();

            while (elementIterator.hasNext()){
                Element node = elementIterator.next();
                String attributesString = node.attributes().toString();
                String text = node.text();
                //Remove unneeded elements
                if (attributesString.contains(" valign=\"top\"") || attributesString.contains(" align=\"left\"")
                        || attributesString.contains(" width=") || attributesString.contains(" colspan=")
                        || text.getBytes().length == 0 || Character.isLetter(text.charAt(0))){
                    //Add our skill names to the map
                    if (attributesString.contains(" align=\"left\"") && !attributesString.contains(" valign=\"top\"")){
                        System.out.println("Text available: " + text);
                        System.out.println("Type added: " + text);
                        typesAvailable.add(text);
                        statsHashMap.put(text, new Stats(0, 0, 0));
                    }
                    System.out.println("Removed: " + node.attributes());
                    elementIterator.remove();
                }
            }

            System.out.println("\n" + "Personal scores for: " + searchPhrase);
            String type, rank, level, xp;
            //Loop through our elements and submit innertext to statsHashMap
            for (int i = 0, j = 0; i < rows.size() && j < typesAvailable.size(); i += 3, j++) {
                //Get type from typesAvailable
                type = typesAvailable.get(j);
                //Remove commas from our innertext data
                rank = rows.get(i).text().replaceAll(",", "");
                level = rows.get(i + 1).text().replaceAll(",", "");
                xp = rows.get(i + 2).text().replaceAll(",", "");
                //Debug
                System.out.println("Type: " + type);
                System.out.println("Rank: " + rank);
                System.out.println("Level: " + level);
                System.out.println("Xp: " + xp + "\n");
                //Add to our stats hashmap
                statsHashMap.put(type, new Stats(Integer.parseInt(rank), Integer.parseInt(level), Integer.parseInt(xp)));
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

        return statsHashMap;
    }

    public static class Stats {
        private int rank;
        private int level;
        private int exp;

        public Stats(int rank, int level, int exp) {
            this.rank = rank;
            this.level = level;
            this.exp = exp;
        }

        public int getRank() {
            return this.rank;
        }

        public int getLevel() {
            return this.level;
        }

        public int getExp() {
            return this.exp;
        }
    }

    //This uses Jaunt API but does not work inside OSbot scripts! Still was fun to learn because the API can fill forms and such
    /*public static HashMap<String, Stats> find(String searchPhrase){
        HashMap<String, Stats> statsHashMap = new HashMap<>();
        try {
            /*  API: http://jaunt-api.com/javadocs/index.html
                Source: http://jaunt-api.com/
                Usage: Import Source as JAR
                Warning: Ignores Overall statistics

            //Jaunt useragent connection and user submit
            UserAgent userAgent = new UserAgent();
            userAgent.setCacheEnabled(false);
            userAgent.visit("https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws");
            userAgent.doc.apply(searchPhrase);
            userAgent.doc.submit("Search");

            //Find table data elements and populate list
            Elements table = userAgent.doc.findEach("<td align=\"right\">");
            List<Element> elementsList = table.toList();
            List<String> typesAvailable = new ArrayList<>();
            Iterator<Element> elementIterator = elementsList.iterator();
            userAgent.close();

            //Remove unneeded elements from our List<Element> elementsList
            while (elementIterator.hasNext()){
                Element element = elementIterator.next();
                String innerText = element.innerText();
                String innerHTML = element.innerHTML();

                if (innerText.getBytes().length == 0 || Character.isLetter(innerText.charAt(0))){
                    //Grab available types if any (e.g Attack)
                    if (innerHTML.contains("src=")){
                        //Format: <img class="miniimg" src="https://www.runescape.com/img/rsp777/hiscores/skill_icon_attack1.gif">
                        String tableName = innerHTML.substring(innerHTML.indexOf("icon_") + 5, innerHTML.indexOf("1"));
                        System.out.println("Added type: " + tableName);
                        typesAvailable.add((tableName.substring(0, 1).toUpperCase() + tableName.substring(1)));
                    }
                    System.out.println("Removed: " + innerText);
                    elementIterator.remove();
                }

            }

            System.out.println("\n" + "Personal scores for: " + searchPhrase);
            String type, rank, level, xp;
            //Loop through our elements and submit innertext to statsHashMap
            for (int i = 0, j = 0; i < elementsList.size() && j < typesAvailable.size(); i += 3, j++) {
                //Get type from typesAvailable
                type = typesAvailable.get(j);
                //Remove commas from our innertext data
                rank = elementsList.get(i).innerText().replaceAll(",", "");
                level = elementsList.get(i + 1).innerText().replaceAll(",", "");
                xp = elementsList.get(i + 2).innerText().replaceAll(",", "");
                //Debug
                System.out.println("Type: " + type);
                System.out.println("Rank: " + rank);
                System.out.println("Level: " + level);
                System.out.println("Xp: " + xp + "\n");
                //Add to our stats hashmap
                statsHashMap.put(type, new Stats(Integer.parseInt(rank), Integer.parseInt(level), Integer.parseInt(xp)));
            }

            System.out.println("Skills map size: " + statsHashMap.size());

        } catch (JauntException | IOException e) {
            e.printStackTrace();
        }

        return statsHashMap;
    }      */

}

Explv · September 23, 2018

Good job, proud of u

Chris · September 23, 2018

56 minutes ago, Explv said:

Good job, proud of u

Learned how to use Jaunt & JSoup :doge:

*SpritexCodeine* · September 23, 2018

impressive, thanks a lot!!

liverare · September 26, 2018

Good work!

You can achieve the same results using Regex - and you should. JSOUP isn't part of the standard Java distribution and it's not bundled within OSBot, so that means if you wanted to use this API in a script on the SVN, the user would have to download and re-download the entire JSOUP library each time. That's quite a bit of overhead.

Edited September 26, 2018 by liverare

Chris · September 28, 2018

On 9/26/2018 at 1:05 PM, liverare said:

Good work!

You can achieve the same results using Regex - and you should. JSOUP isn't part of the standard Java distribution and it's not bundled within OSBot, so that means if you wanted to use this API in a script on the SVN, the user would have to download and re-download the entire JSOUP library each time. That's quite a bit of overhead.

its more of a learning resource.

liverare · September 28, 2018

1 minute ago, Chris said:

its more of a learning resource.

Fair enough. I decided to go ahead and build a lookup using regex just because I find regex challenging. I managed to get the core functionality working. Perhaps you can make it into something nicer?

Source:

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class HiscoresLookup {

	static String webpage;

	private static final String HISCORES_WEB_ADDRESS = "https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws";
	private static final String FAKE_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
	private static final String SKILL_STATISTIC_MATCH;
	private static final String ACHIEVEMENT_STATISTIC_MATCH;

	static {
		
		StringBuilder sb;
		
		{ /* Match skills */
			sb = new StringBuilder();

			sb.append("<tr>.*?");
			sb.append("<td align=\"left\"><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			SKILL_STATISTIC_MATCH = sb.toString();
		}
		
		{ /* Match other (clues, bounty, etc.) */
			sb = new StringBuilder();
			
			sb.append("<tr>");
			sb.append("<td align=\"right\">.*?</td>.*?");
			sb.append("<td><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td colspan=\"2\" align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			ACHIEVEMENT_STATISTIC_MATCH = sb.toString();
		}
	}

	public static void main(String[] args) throws IOException {
		webpage = downloadHiscoresWebpage("tove styrke");
		extractSkills();
		extractAchievements();
	}

	private static String downloadHiscoresWebpage(String username) throws IOException {
		username = username.replaceAll("\\s", "+");

		StringBuilder webpage = new StringBuilder();
		URL url = new URL(HISCORES_WEB_ADDRESS);
		
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.setRequestMethod("POST");
		connection.setRequestProperty("User-Agent", FAKE_USER_AGENT);
		connection.setRequestProperty("Content-type", "application/x-www-form-urlencoded");
		connection.setDoInput(true);
		connection.setDoOutput(true);
		
		OutputStream out = connection.getOutputStream();
		out.write(String.format("user1=%s&submit=Search", username).getBytes());
		out.flush();
		out.close();
		
		BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
		in.lines().forEach(webpage::append);
		in.close();
		
		return webpage.toString();
	}

	private static void extractSkills() {
		
		System.out.println(extractSkill("Overall"));
		System.out.println(extractSkill("Attack"));
		System.out.println(extractSkill("Defence"));
		System.out.println(extractSkill("Strength"));
		System.out.println(extractSkill("Hitpoints"));
		System.out.println(extractSkill("Ranged"));
		System.out.println(extractSkill("Prayer"));
		System.out.println(extractSkill("Magic"));
		System.out.println(extractSkill("Cooking"));
		System.out.println(extractSkill("Woodcutting"));
		System.out.println(extractSkill("Fletching"));
		System.out.println(extractSkill("Fishing"));
		System.out.println(extractSkill("Firemaking"));
		System.out.println(extractSkill("Crafting"));
		System.out.println(extractSkill("Smithing"));
		System.out.println(extractSkill("Mining"));
		System.out.println(extractSkill("Herblore"));
		System.out.println(extractSkill("Agility"));
		System.out.println(extractSkill("Thieving"));
		System.out.println(extractSkill("Slayer"));
		System.out.println(extractSkill("Farming"));
		System.out.println(extractSkill("Runecraft"));
		System.out.println(extractSkill("Hunter"));
		System.out.println(extractSkill("Construction"));
	}

	private static SkillStatistic extractSkill(String name) throws NumberFormatException {
		
		SkillStatistic statistic = null;
		
		int rank;
		int level;
		long experience;
		
		Pattern p = Pattern.compile(String.format(SKILL_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);
		
		if (m.find()) {
			
			rank       = Integer.parseInt(m.group(2).replaceAll(",", ""));
			level      = Integer.parseInt(m.group(3).replaceAll(",", ""));
			experience = Long.parseLong(m.group(4).replaceAll(",", ""));
			
			statistic = new SkillStatistic(rank, level, experience);
		}
		
		return statistic;
	}

	private static void extractAchievements() {
		
		System.out.println(extractAchivement("Clue Scrolls \\(easy\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(medium\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(hard\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(elite\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(master\\)"));
		System.out.println(extractAchivement("LMS - Rank"));
		System.out.println(extractAchivement("Bounty Hunter - Rogue"));
		System.out.println(extractAchivement("Bounty Hunter - Hunter"));
	}

	private static AchievementStatistic extractAchivement(String name) throws NumberFormatException {

		AchievementStatistic statistic = null;

		int rank;
		int score;

		Pattern p = Pattern.compile(String.format(ACHIEVEMENT_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);

		if (m.find()) {

			rank = Integer.parseInt(m.group(2).replaceAll(",", ""));
			score = Integer.parseInt(m.group(3).replaceAll(",", ""));

			statistic = new AchievementStatistic(rank, score);
		}

		return statistic;
	}

	public static class SkillStatistic implements Statistic {

		private final int rank;
		private final int level;
		private final long experience;

		public SkillStatistic(int rank, int level, long experience) {
			super();
			this.rank = rank;
			this.level = level;
			this.experience = experience;
		}

		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"level\": %s, \"experience\": %s }", rank, level, experience);
		}

		@Override
		public int getRank() {
			return rank;
		}

		@Override
		public int getAmount() {
			return level;
		}

		public long getExperience() {
			return experience;
		}
		
	}
	
	public static class AchievementStatistic implements Statistic {
		
		private final int rank;
		private final int score;
		
		public AchievementStatistic(int rank, int score) {
			super();
			this.rank = rank;
			this.score = score;
		}
		
		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"score\": %s }", rank, score);
		}
		
		@Override
		public int getRank() {
			return rank;
		}
		
		@Override
		public int getAmount() {
			return score;
		}
		
	}
	
	public static interface Statistic {
		
		int getRank();
		
		int getAmount();
	}
}

Chris · September 28, 2018

7 minutes ago, liverare said:

Fair enough. I decided to go ahead and build a lookup using regex just because I find regex challenging. I managed to get the core functionality working. Perhaps you can make it into something nicer?

Source:

Reveal hidden contents



import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class HiscoresLookup {

	static String webpage;

	private static final String HISCORES_WEB_ADDRESS = "https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws";
	private static final String FAKE_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
	private static final String SKILL_STATISTIC_MATCH;
	private static final String ACHIEVEMENT_STATISTIC_MATCH;

	static {
		
		StringBuilder sb;
		
		{ /* Match skills */
			sb = new StringBuilder();

			sb.append("<tr>.*?");
			sb.append("<td align=\"left\"><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			SKILL_STATISTIC_MATCH = sb.toString();
		}
		
		{ /* Match other (clues, bounty, etc.) */
			sb = new StringBuilder();
			
			sb.append("<tr>");
			sb.append("<td align=\"right\">.*?</td>.*?");
			sb.append("<td><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td colspan=\"2\" align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			ACHIEVEMENT_STATISTIC_MATCH = sb.toString();
		}
	}

	public static void main(String[] args) throws IOException {
		webpage = downloadHiscoresWebpage("tove styrke");
		extractSkills();
		extractAchievements();
	}

	private static String downloadHiscoresWebpage(String username) throws IOException {
		username = username.replaceAll("\\s", "+");

		StringBuilder webpage = new StringBuilder();
		URL url = new URL(HISCORES_WEB_ADDRESS);
		
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.setRequestMethod("POST");
		connection.setRequestProperty("User-Agent", FAKE_USER_AGENT);
		connection.setRequestProperty("Content-type", "application/x-www-form-urlencoded");
		connection.setDoInput(true);
		connection.setDoOutput(true);
		
		OutputStream out = connection.getOutputStream();
		out.write(String.format("user1=%s&submit=Search", username).getBytes());
		out.flush();
		out.close();
		
		BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
		in.lines().forEach(webpage::append);
		in.close();
		
		return webpage.toString();
	}

	private static void extractSkills() {
		
		System.out.println(extractSkill("Overall"));
		System.out.println(extractSkill("Attack"));
		System.out.println(extractSkill("Defence"));
		System.out.println(extractSkill("Strength"));
		System.out.println(extractSkill("Hitpoints"));
		System.out.println(extractSkill("Ranged"));
		System.out.println(extractSkill("Prayer"));
		System.out.println(extractSkill("Magic"));
		System.out.println(extractSkill("Cooking"));
		System.out.println(extractSkill("Woodcutting"));
		System.out.println(extractSkill("Fletching"));
		System.out.println(extractSkill("Fishing"));
		System.out.println(extractSkill("Firemaking"));
		System.out.println(extractSkill("Crafting"));
		System.out.println(extractSkill("Smithing"));
		System.out.println(extractSkill("Mining"));
		System.out.println(extractSkill("Herblore"));
		System.out.println(extractSkill("Agility"));
		System.out.println(extractSkill("Thieving"));
		System.out.println(extractSkill("Slayer"));
		System.out.println(extractSkill("Farming"));
		System.out.println(extractSkill("Runecraft"));
		System.out.println(extractSkill("Hunter"));
		System.out.println(extractSkill("Construction"));
	}

	private static SkillStatistic extractSkill(String name) throws NumberFormatException {
		
		SkillStatistic statistic = null;
		
		int rank;
		int level;
		long experience;
		
		Pattern p = Pattern.compile(String.format(SKILL_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);
		
		if (m.find()) {
			
			rank       = Integer.parseInt(m.group(2).replaceAll(",", ""));
			level      = Integer.parseInt(m.group(3).replaceAll(",", ""));
			experience = Long.parseLong(m.group(4).replaceAll(",", ""));
			
			statistic = new SkillStatistic(rank, level, experience);
		}
		
		return statistic;
	}

	private static void extractAchievements() {
		
		System.out.println(extractAchivement("Clue Scrolls \\(easy\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(medium\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(hard\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(elite\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(master\\)"));
		System.out.println(extractAchivement("LMS - Rank"));
		System.out.println(extractAchivement("Bounty Hunter - Rogue"));
		System.out.println(extractAchivement("Bounty Hunter - Hunter"));
	}

	private static AchievementStatistic extractAchivement(String name) throws NumberFormatException {

		AchievementStatistic statistic = null;

		int rank;
		int score;

		Pattern p = Pattern.compile(String.format(ACHIEVEMENT_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);

		if (m.find()) {

			rank = Integer.parseInt(m.group(2).replaceAll(",", ""));
			score = Integer.parseInt(m.group(3).replaceAll(",", ""));

			statistic = new AchievementStatistic(rank, score);
		}

		return statistic;
	}

	public static class SkillStatistic implements Statistic {

		private final int rank;
		private final int level;
		private final long experience;

		public SkillStatistic(int rank, int level, long experience) {
			super();
			this.rank = rank;
			this.level = level;
			this.experience = experience;
		}

		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"level\": %s, \"experience\": %s }", rank, level, experience);
		}

		@Override
		public int getRank() {
			return rank;
		}

		@Override
		public int getAmount() {
			return level;
		}

		public long getExperience() {
			return experience;
		}
		
	}
	
	public static class AchievementStatistic implements Statistic {
		
		private final int rank;
		private final int score;
		
		public AchievementStatistic(int rank, int score) {
			super();
			this.rank = rank;
			this.score = score;
		}
		
		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"score\": %s }", rank, score);
		}
		
		@Override
		public int getRank() {
			return rank;
		}
		
		@Override
		public int getAmount() {
			return score;
		}
		
	}
	
	public static interface Statistic {
		
		int getRank();
		
		int getAmount();
	}
}

Nice

I found this https://secure.runescape.com/m=hiscore_oldschool/index_lite.ws?player=Zezima

so i guess regex would work nicely here?

liverare · September 28, 2018

3 minutes ago, Chris said:

Nice

I found this https://secure.runescape.com/m=hiscore_oldschool/index_lite.ws?player=Zezima

so i guess regex would work nicely here?

Possibly. I wasn't aware Jagex had an API for it - that must be fairly new. You could use a string split regex, to group things by levels. However, I'm not entirely sure on the format until I compare it with the hiscores.

You should redo your API using Jagex's API - it'd save so much overhead.

Sign In

[JSOUP] Hiscores Lookup Snippet

Recommended Posts

Chris

Explv

Chris

SpritexCodeine

liverare

Chris

liverare

Chris

liverare

Create an account or sign in to comment

Create an account

Sign in

Recently Browsing 0 members

Browse

Store

Search