[JSOUP] Hiscores Lookup Snippet

Chris · September 23, 2018


import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.*;

public class HiScores {

    public static HashMap<String, Stats> find(String searchPhrase){
        HashMap<String, Stats> statsHashMap = new HashMap<>();

        try {
            //Jsoup closes the connection by its own, after the request is done
            Document document = Jsoup
                    .connect("https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws?user1=" + searchPhrase.replaceAll(" ", "%20"))
                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")
                    .timeout(3000)
                    .get();
            //Select the first table.
            Element table = document.select("table").get(0);
            //Grab our rows of td from Element table
            Elements rows = table.select("td");
            //Iterator to remove unnecessary data
            Iterator<Element> elementIterator = rows.iterator();
            //List holds our skill type names
            List<String> typesAvailable = new ArrayList<>();

            while (elementIterator.hasNext()){
                Element node = elementIterator.next();
                String attributesString = node.attributes().toString();
                String text = node.text();
                //Remove unneeded elements
                if (attributesString.contains(" valign=\"top\"") || attributesString.contains(" align=\"left\"")
                        || attributesString.contains(" width=") || attributesString.contains(" colspan=")
                        || text.getBytes().length == 0 || Character.isLetter(text.charAt(0))){
                    //Add our skill names to the map
                    if (attributesString.contains(" align=\"left\"") && !attributesString.contains(" valign=\"top\"")){
                        System.out.println("Text available: " + text);
                        System.out.println("Type added: " + text);
                        typesAvailable.add(text);
                        statsHashMap.put(text, new Stats(0, 0, 0));
                    }
                    System.out.println("Removed: " + node.attributes());
                    elementIterator.remove();
                }
            }

            System.out.println("\n" + "Personal scores for: " + searchPhrase);
            String type, rank, level, xp;
            //Loop through our elements and submit innertext to statsHashMap
            for (int i = 0, j = 0; i < rows.size() && j < typesAvailable.size(); i += 3, j++) {
                //Get type from typesAvailable
                type = typesAvailable.get(j);
                //Remove commas from our innertext data
                rank = rows.get(i).text().replaceAll(",", "");
                level = rows.get(i + 1).text().replaceAll(",", "");
                xp = rows.get(i + 2).text().replaceAll(",", "");
                //Debug
                System.out.println("Type: " + type);
                System.out.println("Rank: " + rank);
                System.out.println("Level: " + level);
                System.out.println("Xp: " + xp + "\n");
                //Add to our stats hashmap
                statsHashMap.put(type, new Stats(Integer.parseInt(rank), Integer.parseInt(level), Integer.parseInt(xp)));
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

        return statsHashMap;
    }

    public static class Stats {
        private int rank;
        private int level;
        private int exp;

        public Stats(int rank, int level, int exp) {
            this.rank = rank;
            this.level = level;
            this.exp = exp;
        }

        public int getRank() {
            return this.rank;
        }

        public int getLevel() {
            return this.level;
        }

        public int getExp() {
            return this.exp;
        }
    }

    //This uses Jaunt API but does not work inside OSbot scripts! Still was fun to learn because the API can fill forms and such
    /*public static HashMap<String, Stats> find(String searchPhrase){
        HashMap<String, Stats> statsHashMap = new HashMap<>();
        try {
            /*  API: http://jaunt-api.com/javadocs/index.html
                Source: http://jaunt-api.com/
                Usage: Import Source as JAR
                Warning: Ignores Overall statistics

            //Jaunt useragent connection and user submit
            UserAgent userAgent = new UserAgent();
            userAgent.setCacheEnabled(false);
            userAgent.visit("https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws");
            userAgent.doc.apply(searchPhrase);
            userAgent.doc.submit("Search");

            //Find table data elements and populate list
            Elements table = userAgent.doc.findEach("<td align=\"right\">");
            List<Element> elementsList = table.toList();
            List<String> typesAvailable = new ArrayList<>();
            Iterator<Element> elementIterator = elementsList.iterator();
            userAgent.close();

            //Remove unneeded elements from our List<Element> elementsList
            while (elementIterator.hasNext()){
                Element element = elementIterator.next();
                String innerText = element.innerText();
                String innerHTML = element.innerHTML();

                if (innerText.getBytes().length == 0 || Character.isLetter(innerText.charAt(0))){
                    //Grab available types if any (e.g Attack)
                    if (innerHTML.contains("src=")){
                        //Format: <img class="miniimg" src="https://www.runescape.com/img/rsp777/hiscores/skill_icon_attack1.gif">
                        String tableName = innerHTML.substring(innerHTML.indexOf("icon_") + 5, innerHTML.indexOf("1"));
                        System.out.println("Added type: " + tableName);
                        typesAvailable.add((tableName.substring(0, 1).toUpperCase() + tableName.substring(1)));
                    }
                    System.out.println("Removed: " + innerText);
                    elementIterator.remove();
                }

            }

            System.out.println("\n" + "Personal scores for: " + searchPhrase);
            String type, rank, level, xp;
            //Loop through our elements and submit innertext to statsHashMap
            for (int i = 0, j = 0; i < elementsList.size() && j < typesAvailable.size(); i += 3, j++) {
                //Get type from typesAvailable
                type = typesAvailable.get(j);
                //Remove commas from our innertext data
                rank = elementsList.get(i).innerText().replaceAll(",", "");
                level = elementsList.get(i + 1).innerText().replaceAll(",", "");
                xp = elementsList.get(i + 2).innerText().replaceAll(",", "");
                //Debug
                System.out.println("Type: " + type);
                System.out.println("Rank: " + rank);
                System.out.println("Level: " + level);
                System.out.println("Xp: " + xp + "\n");
                //Add to our stats hashmap
                statsHashMap.put(type, new Stats(Integer.parseInt(rank), Integer.parseInt(level), Integer.parseInt(xp)));
            }

            System.out.println("Skills map size: " + statsHashMap.size());

        } catch (JauntException | IOException e) {
            e.printStackTrace();
        }

        return statsHashMap;
    }      */

}

Explv · September 23, 2018

Good job, proud of u

Chris · September 23, 2018

56 minutes ago, Explv said:

Good job, proud of u

Learned how to use Jaunt & JSoup :doge:

*SpritexCodeine* · September 23, 2018

impressive, thanks a lot!!

liverare · September 26, 2018

Good work!

You can achieve the same results using Regex - and you should. JSOUP isn't part of the standard Java distribution and it's not bundled within OSBot, so that means if you wanted to use this API in a script on the SVN, the user would have to download and re-download the entire JSOUP library each time. That's quite a bit of overhead.

Edited September 26, 2018 by liverare

Chris · September 28, 2018

On 9/26/2018 at 1:05 PM, liverare said:

Good work!

You can achieve the same results using Regex - and you should. JSOUP isn't part of the standard Java distribution and it's not bundled within OSBot, so that means if you wanted to use this API in a script on the SVN, the user would have to download and re-download the entire JSOUP library each time. That's quite a bit of overhead.

its more of a learning resource.

liverare · September 28, 2018

1 minute ago, Chris said:

its more of a learning resource.

Fair enough. I decided to go ahead and build a lookup using regex just because I find regex challenging. I managed to get the core functionality working. Perhaps you can make it into something nicer?

Source:

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class HiscoresLookup {

	static String webpage;

	private static final String HISCORES_WEB_ADDRESS = "https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws";
	private static final String FAKE_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
	private static final String SKILL_STATISTIC_MATCH;
	private static final String ACHIEVEMENT_STATISTIC_MATCH;

	static {
		
		StringBuilder sb;
		
		{ /* Match skills */
			sb = new StringBuilder();

			sb.append("<tr>.*?");
			sb.append("<td align=\"left\"><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			SKILL_STATISTIC_MATCH = sb.toString();
		}
		
		{ /* Match other (clues, bounty, etc.) */
			sb = new StringBuilder();
			
			sb.append("<tr>");
			sb.append("<td align=\"right\">.*?</td>.*?");
			sb.append("<td><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td colspan=\"2\" align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			ACHIEVEMENT_STATISTIC_MATCH = sb.toString();
		}
	}

	public static void main(String[] args) throws IOException {
		webpage = downloadHiscoresWebpage("tove styrke");
		extractSkills();
		extractAchievements();
	}

	private static String downloadHiscoresWebpage(String username) throws IOException {
		username = username.replaceAll("\\s", "+");

		StringBuilder webpage = new StringBuilder();
		URL url = new URL(HISCORES_WEB_ADDRESS);
		
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.setRequestMethod("POST");
		connection.setRequestProperty("User-Agent", FAKE_USER_AGENT);
		connection.setRequestProperty("Content-type", "application/x-www-form-urlencoded");
		connection.setDoInput(true);
		connection.setDoOutput(true);
		
		OutputStream out = connection.getOutputStream();
		out.write(String.format("user1=%s&submit=Search", username).getBytes());
		out.flush();
		out.close();
		
		BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
		in.lines().forEach(webpage::append);
		in.close();
		
		return webpage.toString();
	}

	private static void extractSkills() {
		
		System.out.println(extractSkill("Overall"));
		System.out.println(extractSkill("Attack"));
		System.out.println(extractSkill("Defence"));
		System.out.println(extractSkill("Strength"));
		System.out.println(extractSkill("Hitpoints"));
		System.out.println(extractSkill("Ranged"));
		System.out.println(extractSkill("Prayer"));
		System.out.println(extractSkill("Magic"));
		System.out.println(extractSkill("Cooking"));
		System.out.println(extractSkill("Woodcutting"));
		System.out.println(extractSkill("Fletching"));
		System.out.println(extractSkill("Fishing"));
		System.out.println(extractSkill("Firemaking"));
		System.out.println(extractSkill("Crafting"));
		System.out.println(extractSkill("Smithing"));
		System.out.println(extractSkill("Mining"));
		System.out.println(extractSkill("Herblore"));
		System.out.println(extractSkill("Agility"));
		System.out.println(extractSkill("Thieving"));
		System.out.println(extractSkill("Slayer"));
		System.out.println(extractSkill("Farming"));
		System.out.println(extractSkill("Runecraft"));
		System.out.println(extractSkill("Hunter"));
		System.out.println(extractSkill("Construction"));
	}

	private static SkillStatistic extractSkill(String name) throws NumberFormatException {
		
		SkillStatistic statistic = null;
		
		int rank;
		int level;
		long experience;
		
		Pattern p = Pattern.compile(String.format(SKILL_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);
		
		if (m.find()) {
			
			rank       = Integer.parseInt(m.group(2).replaceAll(",", ""));
			level      = Integer.parseInt(m.group(3).replaceAll(",", ""));
			experience = Long.parseLong(m.group(4).replaceAll(",", ""));
			
			statistic = new SkillStatistic(rank, level, experience);
		}
		
		return statistic;
	}

	private static void extractAchievements() {
		
		System.out.println(extractAchivement("Clue Scrolls \\(easy\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(medium\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(hard\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(elite\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(master\\)"));
		System.out.println(extractAchivement("LMS - Rank"));
		System.out.println(extractAchivement("Bounty Hunter - Rogue"));
		System.out.println(extractAchivement("Bounty Hunter - Hunter"));
	}

	private static AchievementStatistic extractAchivement(String name) throws NumberFormatException {

		AchievementStatistic statistic = null;

		int rank;
		int score;

		Pattern p = Pattern.compile(String.format(ACHIEVEMENT_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);

		if (m.find()) {

			rank = Integer.parseInt(m.group(2).replaceAll(",", ""));
			score = Integer.parseInt(m.group(3).replaceAll(",", ""));

			statistic = new AchievementStatistic(rank, score);
		}

		return statistic;
	}

	public static class SkillStatistic implements Statistic {

		private final int rank;
		private final int level;
		private final long experience;

		public SkillStatistic(int rank, int level, long experience) {
			super();
			this.rank = rank;
			this.level = level;
			this.experience = experience;
		}

		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"level\": %s, \"experience\": %s }", rank, level, experience);
		}

		@Override
		public int getRank() {
			return rank;
		}

		@Override
		public int getAmount() {
			return level;
		}

		public long getExperience() {
			return experience;
		}
		
	}
	
	public static class AchievementStatistic implements Statistic {
		
		private final int rank;
		private final int score;
		
		public AchievementStatistic(int rank, int score) {
			super();
			this.rank = rank;
			this.score = score;
		}
		
		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"score\": %s }", rank, score);
		}
		
		@Override
		public int getRank() {
			return rank;
		}
		
		@Override
		public int getAmount() {
			return score;
		}
		
	}
	
	public static interface Statistic {
		
		int getRank();
		
		int getAmount();
	}
}

Chris · September 28, 2018

7 minutes ago, liverare said:

Fair enough. I decided to go ahead and build a lookup using regex just because I find regex challenging. I managed to get the core functionality working. Perhaps you can make it into something nicer?

Source:

Reveal hidden contents



import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class HiscoresLookup {

	static String webpage;

	private static final String HISCORES_WEB_ADDRESS = "https://secure.runescape.com/m=hiscore_oldschool/hiscorepersonal.ws";
	private static final String FAKE_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
	private static final String SKILL_STATISTIC_MATCH;
	private static final String ACHIEVEMENT_STATISTIC_MATCH;

	static {
		
		StringBuilder sb;
		
		{ /* Match skills */
			sb = new StringBuilder();

			sb.append("<tr>.*?");
			sb.append("<td align=\"left\"><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			SKILL_STATISTIC_MATCH = sb.toString();
		}
		
		{ /* Match other (clues, bounty, etc.) */
			sb = new StringBuilder();
			
			sb.append("<tr>");
			sb.append("<td align=\"right\">.*?</td>.*?");
			sb.append("<td><a[^>]*>(%s)</a></td>");
			sb.append("<td align=\"right\">(.*?)</td>");
			sb.append("<td colspan=\"2\" align=\"right\">(.*?)</td>");
			sb.append("</tr>");
			
			ACHIEVEMENT_STATISTIC_MATCH = sb.toString();
		}
	}

	public static void main(String[] args) throws IOException {
		webpage = downloadHiscoresWebpage("tove styrke");
		extractSkills();
		extractAchievements();
	}

	private static String downloadHiscoresWebpage(String username) throws IOException {
		username = username.replaceAll("\\s", "+");

		StringBuilder webpage = new StringBuilder();
		URL url = new URL(HISCORES_WEB_ADDRESS);
		
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.setRequestMethod("POST");
		connection.setRequestProperty("User-Agent", FAKE_USER_AGENT);
		connection.setRequestProperty("Content-type", "application/x-www-form-urlencoded");
		connection.setDoInput(true);
		connection.setDoOutput(true);
		
		OutputStream out = connection.getOutputStream();
		out.write(String.format("user1=%s&submit=Search", username).getBytes());
		out.flush();
		out.close();
		
		BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
		in.lines().forEach(webpage::append);
		in.close();
		
		return webpage.toString();
	}

	private static void extractSkills() {
		
		System.out.println(extractSkill("Overall"));
		System.out.println(extractSkill("Attack"));
		System.out.println(extractSkill("Defence"));
		System.out.println(extractSkill("Strength"));
		System.out.println(extractSkill("Hitpoints"));
		System.out.println(extractSkill("Ranged"));
		System.out.println(extractSkill("Prayer"));
		System.out.println(extractSkill("Magic"));
		System.out.println(extractSkill("Cooking"));
		System.out.println(extractSkill("Woodcutting"));
		System.out.println(extractSkill("Fletching"));
		System.out.println(extractSkill("Fishing"));
		System.out.println(extractSkill("Firemaking"));
		System.out.println(extractSkill("Crafting"));
		System.out.println(extractSkill("Smithing"));
		System.out.println(extractSkill("Mining"));
		System.out.println(extractSkill("Herblore"));
		System.out.println(extractSkill("Agility"));
		System.out.println(extractSkill("Thieving"));
		System.out.println(extractSkill("Slayer"));
		System.out.println(extractSkill("Farming"));
		System.out.println(extractSkill("Runecraft"));
		System.out.println(extractSkill("Hunter"));
		System.out.println(extractSkill("Construction"));
	}

	private static SkillStatistic extractSkill(String name) throws NumberFormatException {
		
		SkillStatistic statistic = null;
		
		int rank;
		int level;
		long experience;
		
		Pattern p = Pattern.compile(String.format(SKILL_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);
		
		if (m.find()) {
			
			rank       = Integer.parseInt(m.group(2).replaceAll(",", ""));
			level      = Integer.parseInt(m.group(3).replaceAll(",", ""));
			experience = Long.parseLong(m.group(4).replaceAll(",", ""));
			
			statistic = new SkillStatistic(rank, level, experience);
		}
		
		return statistic;
	}

	private static void extractAchievements() {
		
		System.out.println(extractAchivement("Clue Scrolls \\(easy\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(medium\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(hard\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(elite\\)"));
		System.out.println(extractAchivement("Clue Scrolls \\(master\\)"));
		System.out.println(extractAchivement("LMS - Rank"));
		System.out.println(extractAchivement("Bounty Hunter - Rogue"));
		System.out.println(extractAchivement("Bounty Hunter - Hunter"));
	}

	private static AchievementStatistic extractAchivement(String name) throws NumberFormatException {

		AchievementStatistic statistic = null;

		int rank;
		int score;

		Pattern p = Pattern.compile(String.format(ACHIEVEMENT_STATISTIC_MATCH, name));
		Matcher m = p.matcher(webpage);

		if (m.find()) {

			rank = Integer.parseInt(m.group(2).replaceAll(",", ""));
			score = Integer.parseInt(m.group(3).replaceAll(",", ""));

			statistic = new AchievementStatistic(rank, score);
		}

		return statistic;
	}

	public static class SkillStatistic implements Statistic {

		private final int rank;
		private final int level;
		private final long experience;

		public SkillStatistic(int rank, int level, long experience) {
			super();
			this.rank = rank;
			this.level = level;
			this.experience = experience;
		}

		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"level\": %s, \"experience\": %s }", rank, level, experience);
		}

		@Override
		public int getRank() {
			return rank;
		}

		@Override
		public int getAmount() {
			return level;
		}

		public long getExperience() {
			return experience;
		}
		
	}
	
	public static class AchievementStatistic implements Statistic {
		
		private final int rank;
		private final int score;
		
		public AchievementStatistic(int rank, int score) {
			super();
			this.rank = rank;
			this.score = score;
		}
		
		@Override
		public String toString() {
			return String.format("{ \"rank\": %s, \"score\": %s }", rank, score);
		}
		
		@Override
		public int getRank() {
			return rank;
		}
		
		@Override
		public int getAmount() {
			return score;
		}
		
	}
	
	public static interface Statistic {
		
		int getRank();
		
		int getAmount();
	}
}

Nice

I found this https://secure.runescape.com/m=hiscore_oldschool/index_lite.ws?player=Zezima

so i guess regex would work nicely here?

liverare · September 28, 2018

3 minutes ago, Chris said:

Nice

I found this https://secure.runescape.com/m=hiscore_oldschool/index_lite.ws?player=Zezima

so i guess regex would work nicely here?

Possibly. I wasn't aware Jagex had an API for it - that must be fairly new. You could use a string split regex, to group things by levels. However, I'm not entirely sure on the format until I compare it with the hiscores.

You should redo your API using Jagex's API - it'd save so much overhead.

Sign In

[JSOUP] Hiscores Lookup Snippet

Recommended Posts

Chris

Link to comment

Share on other sites

Explv

Link to comment

Share on other sites

Chris

Link to comment

Share on other sites

SpritexCodeine

Link to comment

Share on other sites

liverare

Link to comment

Share on other sites

Chris

Link to comment

Share on other sites

liverare

Link to comment

Share on other sites

Chris

Link to comment

Share on other sites

liverare

Link to comment

Share on other sites

Join the conversation

Recently Browsing 0 members

Browse

Store

Search