[Userscript] WaniKani Open Framework Additional Filters (Recent Lessons, Leech Training, Related Items, and more)

I read your post in the APIv2 Thread.
Have you found a reliable way?

This one can probably be done now, I just have to figure out the specifics. I might not get to it for a few days.

I could build a prototype for you. Just send me which endpoint fields are necessary and I figure it out :slight_smile:

@seanblue

// Edited -> Check Below.

Here I made the next function work. Take it or parts of it how you like :slight_smile:

The option defines the maximum amount of days which can pass until they don’t show up in the quiz

This doesn’t do what I had in mind for the filter. I’m not seeing what isAtLeastMinimumDaysUntilReview has to do with recently failed reviews. But maybe I’m missing something…

I’m sorry but I don’t know what you have in mind @seanblue :hugs:
About the isAtLeastMinimumDaysUntilReview. Well I forgot to rename the function, the curse of copy and paste mixed with bedtime, sorry :disappointed:. I changed it a little bit. I hope its clear now :slight_smile:

// ==UserScript==
// @name          WaniKani Open Framework Additional Filters
// @namespace     https://www.wanikani.com
// @description   Additional filters for the WaniKani Open Framework
// @author        seanblue
// @version       1.1.0
// @include       https://www.wanikani.com/*
// @grant         none
// ==/UserScript==

(function() {
	'use strict';

	var wkofMinimumVersion = '1.0.18';

	if (!window.wkof) {
		alert('WaniKani Open Framework Additional Filters requires WaniKani Open Framework.\nYou will now be forwarded to installation instructions.');
		window.location.href = 'https://community.wanikani.com/t/instructions-installing-wanikani-open-framework/28549';
		return;
	}

	if (!wkof.version || wkof.version.compare_to(wkofMinimumVersion) === 'older') {
		alert('WaniKani Open Framework Additional Filters requires at least version ' + wkofMinimumVersion + ' of WaniKani Open Framework.');
		return;
	}

	var settingsDialog;
	var settingsScriptId = 'additionalFilters';
	var settingsTitle = 'Additional Filters';

	var needToRegisterFilters = true;
	var settingsLoadedPromise = promise();

	var filterNamePrefix = 'additionalFilters_';
	var recentLessonsFilterName = filterNamePrefix + 'recentLessons';
	var leechTrainingFilterName = filterNamePrefix + 'leechTraining';
	var timeUntilReviewFilterName = filterNamePrefix + 'timeUntilReview';
	var recentlyFailedFilterName = filterNamePrefix + 'recentlyFailed';

	var supportedFilters = [recentLessonsFilterName, leechTrainingFilterName, timeUntilReviewFilterName, recentlyFailedFilterName];

	var defaultSettings = {};
	defaultSettings[recentLessonsFilterName] = true;
	defaultSettings[leechTrainingFilterName] = true;
	defaultSettings[timeUntilReviewFilterName] = true;
	defaultSettings[recentlyFailedFilterName] = true;

	var recentLessonsHoverTip = 'Only include lessons taken in the last X hours.';
	var leechesSummaryHoverTip = 'Only include leeches. Formula: incorrect / currentStreak^1.5.';
	var leechesHoverTip = leechesSummaryHoverTip + '\n * The higher the value, the fewer items will be included as leeches.\n * Setting the value to 1 will include items that have just been answered incorrectly for the first time.\n * Setting the value to 1.01 will exclude items that have just been answered incorrectly for the first time.';

	var timeUntilReviewSummaryHoverTip = 'Only include items that have at least X% of their SRS interval remaining.';
	var timeUntilReviewHoverTip = timeUntilReviewSummaryHoverTip + '\nValid values are from 0 to 100. Examples:\n "75": At least 75% of an item\'s SRS interval must be remaining.';

	var recentlyFailedSummaryHoverTip = 'Only include items that have recently Failed.';
	var recentlyFailedHoverTip = recentlyFailedSummaryHoverTip + '\n Only include Items which failed X days ago or less.';

	var msToHoursDivisor = 3600000;
    var msToDaysDivisor = 86400000;

	var nowForTimeUntilReview;
	var regularSrsIntervals = [0, 4, 8, 23, 47, 167, 335, 719, 2879];
	var acceleratedSrsIntervals = [0, 2, 4, 8, 23, 167, 335, 719, 2879];
	var acceleratedLevels = [1, 2];

	wkof.include('Menu, Settings');

	wkof.ready('Menu').then(installMenu);
	waitForItemDataRegistry().then(installSettings);

	function promise(){var a,b,c=new Promise(function(d,e){a=d;b=e;});c.resolve=a;c.reject=b;return c;}

	function waitForItemDataRegistry() {
		return wkof.wait_state('wkof.ItemData.registry', 'ready');
	}

	function installMenu() {
		loadSettings().then(function() {
			addMenuItem();
		});
	}

	function addMenuItem() {
		wkof.Menu.insert_script_link({
			script_id: settingsScriptId,
			submenu: 'Settings',
			title: settingsTitle,
			on_click: function() { settingsDialog.open(); }
		});
	}

	function installSettings() {
		wkof.ItemData.pause_ready_event(true);

		loadSettings().then(function() {
			wkof.ItemData.pause_ready_event(false);
		});
	}

	function loadSettings(postLoadAction) {
		wkof.ready('Settings').then(function() {
			if (settingsDialog) {
				return;
			}

			var settings = {};
			settings[recentLessonsFilterName] = { type: 'checkbox', label: 'Recent Lessons', hover_tip: recentLessonsHoverTip };
			settings[leechTrainingFilterName] = { type: 'checkbox', label: 'Leech Training', hover_tip: leechesSummaryHoverTip };
			settings[timeUntilReviewFilterName] = { type: 'checkbox', label: 'Time Until Review', hover_tip: timeUntilReviewSummaryHoverTip };
			settings[recentlyFailedFilterName] = { type: 'checkbox', label: 'Time Until Review', hover_tip: recentlyFailedSummaryHoverTip };

			settingsDialog = new wkof.Settings({
				script_id: settingsScriptId,
				title: settingsTitle,
				on_save: saveSettings,
				settings: settings
			});

			settingsDialog.load(defaultSettings).then(function() {
				updateFiltersWhenReady();
				settingsLoadedPromise.resolve();
			});
		});

		return settingsLoadedPromise;
	}

	function saveSettings(){
		settingsDialog.save().then(function() {
			updateFiltersWhenReady();
		});
	}

	function updateFiltersWhenReady() {
		needToRegisterFilters = true;
		waitForItemDataRegistry().then(registerFilters);
	}

	function registerFilters() {
		if (!needToRegisterFilters)
			return;

		supportedFilters.forEach(function(filterName) {
			delete wkof.ItemData.registry.sources.wk_items.filters[filterName];
		});

		if (wkof.settings[settingsScriptId][recentLessonsFilterName])
			registerRecentLessonsFilter();

		if (wkof.settings[settingsScriptId][leechTrainingFilterName])
			registerLeechTrainingFilter();

		if (wkof.settings[settingsScriptId][timeUntilReviewFilterName])
			registerTimeUntilReviewFilter();

		if (wkof.settings[settingsScriptId][recentlyFailedFilterName])
			registerRecentlyFailedFilter();

		needToRegisterFilters = false;
	}

	// BEGIN Recent Lessons
	function registerRecentLessonsFilter() {
		wkof.ItemData.registry.sources.wk_items.filters[recentLessonsFilterName] = {
			type: 'number',
			label: 'Recent Lessons',
			default: 24,
			placeholder: '24',
			filter_func: recentLessonsFilter,
			set_options: function(options) { options.assignments = true; },
			hover_tip: recentLessonsHoverTip
		};
	}

	function recentLessonsFilter(filterValue, item) {
		if (item.assignments === undefined)
			return false;

		var startedAt = item.assignments.started_at;
		if (startedAt === null || startedAt === undefined)
			return false;

		var startedAtDate = new Date(startedAt);
		var timeSinceStart = Date.now() - startedAtDate;

		return (timeSinceStart / msToHoursDivisor) < filterValue;
	}
	// END Recent Lessons

	// BEGIN Leeches
	function registerLeechTrainingFilter() {
		wkof.ItemData.registry.sources.wk_items.filters[leechTrainingFilterName] = {
			type: 'number',
			label: 'Leech Training',
			default: 1,
			placeholder: '1',
			filter_func: leechTrainingFilter,
			set_options: function(options) { options.review_statistics = true; },
			hover_tip: leechesHoverTip
		};
	}

	function leechTrainingFilter(filterValue, item) {
		if (item.review_statistics === undefined)
			return false;

		var reviewStats = item.review_statistics;
		var meaningScore = getLeechScore(reviewStats.meaning_incorrect, reviewStats.meaning_current_streak);
		var readingScore = getLeechScore(reviewStats.reading_incorrect, reviewStats.reading_current_streak);

		return meaningScore >= filterValue || readingScore >= filterValue;
	}

	function getLeechScore(incorrect, currentStreak) {
		return incorrect / Math.pow((currentStreak || 0.5), 1.5);
	}
	// END Leeches

	// BEGIN Time Until Review
	function registerTimeUntilReviewFilter() {
		wkof.ItemData.registry.sources.wk_items.filters[timeUntilReviewFilterName] = {
			type: 'number',
			label: 'Time Until Review',
			default: 50,
			placeholder: '50',
			prepare: timeUntilReviewPrepare,
			filter_value_map: timeUntilReviewValueMap,
			filter_func: timeUntilReviewFilter,
			set_options: function(options) { options.assignments = true; },
			hover_tip: timeUntilReviewHoverTip
		};
	}

	function timeUntilReviewPrepare() {
		// Only set "now" once so that all items use the same value when filtering.
		nowForTimeUntilReview = Date.now();
	}

	function timeUntilReviewValueMap(percentage) {
		if (percentage < 0)
			return 0;

		if (percentage > 100)
			return 100;

		return percentage;
	}

	function timeUntilReviewFilter(percentage, item) {
		if (item.assignments === undefined)
			return false;

		var srsStage = item.assignments.srs_stage;
		if (srsStage === 0)
			return false;

		if (srsStage === 9)
			return true;

		var level = item.assignments.level;
		var reviewAvailableAt = item.assignments.available_at;
		return isAtLeastMinimumHoursUntilReview(srsStage, level, reviewAvailableAt, percentage);
	}

	function isAtLeastMinimumHoursUntilReview(srsStage, level, reviewAvailableAt, percentage) {
		var hoursUntilReview = (new Date(reviewAvailableAt).getTime() - nowForTimeUntilReview) / msToHoursDivisor;

		var srsInvervals = acceleratedLevels.includes(level) ? acceleratedSrsIntervals : regularSrsIntervals;
		var minimumHoursUntilReview =  srsInvervals[srsStage] * percentage / 100;

		return minimumHoursUntilReview <= hoursUntilReview;
	}
	// END Time Until Review

    // BEGIN Recently Failed
	function registerRecentlyFailedFilter() {
		wkof.ItemData.registry.sources.wk_items.filters[recentlyFailedFilterName] = {
			type: 'number',
			label: 'Recently Failed',
			default: 2,
			placeholder: '2',
			prepare: timeUntilReviewPrepare,
			filter_func: recentlyFailedFilter,
			set_options: function(options) { options.review_statistics = true; options.assignments = true; options.reviews = true; },
			hover_tip: recentlyFailedHoverTip
		};
	}

	function recentlyFailedFilter(filterValue, item) {
		if (item.assignments === undefined)
			return false;


		var level = item.assignments.level;
		var srsStage = item.assignments.srs_stage;
        var meaningStreak = item.review_statistics.meaning_current_streak;
        var readingStreak = item.review_statistics.reading_current_streak;
		var reviewAvailableAt = item.assignments.available_at;

        if (meaningStreak > 1 && readingStreak > 1)
            return false;

		if (srsStage === 0)
			return false;

		if (srsStage === 9)
			return false;

		var lastReview = getLastReviewDate(srsStage,level, reviewAvailableAt);
		var daysSinceLastReview = (nowForTimeUntilReview - lastReview.getTime()) / msToDaysDivisor;
		return daysSinceLastReview <= filterValue;
	}

	function getLastReviewDate(srsStage,level, reviewAvailableAt) {
		var srsInvervals = acceleratedLevels.includes(level) ? acceleratedSrsIntervals : regularSrsIntervals;
		return new Date(new Date(reviewAvailableAt).getTime() - (srsInvervals[srsStage] * msToHoursDivisor));
	}

	// END  Recently Failed
})();

I see. You’re using the time of the next review to estimate the time of the last review. There’s one big downside to this approach, which is that if you got a review wrong and then you got a later review right, this formula can’t know that. It’ll just see that the streak is now 2.

I’m considering using the reviews endpoint, which should be 100% accurate. But the tradeoff there is that it will be a lot slower because that data isn’t cached.

I’m debating making the “review time period to look at” configurable in my script’s settings, rather than being the input in Self Study. That way that data could be reused for other filters (though I have no other filters using review data in mind right now). And also I could then make other data the Self Study filter criteria, like the number of failures in that time period. So you could say, show all items that have two or more failures over the last X hours.

I’m still not set on that approach though, given the added complexity and performance concerns. Do you think that approach would be more or less valuable than the current streak approach?

You have a point there. My goal was only the last review.

I’m always for more options. So it would have more value.
But, maybe you should ask the community if its worth the effort, it looks more like something I would use in a statistic.

Your example: “which is that if you got a review wrong and then you got a later review right, this formula can’t know that”

If I get it right why would I want to get more exposure.
If its a leech, the leech filter would deal with it.

I don’t know enough to give an opinion to this.

That’s a good point. Maybe failed last review is sufficient.

If anyone else has an opinion on this, chime in in the next few days. I’m probably going to implement it this weekend.

If I were coding a general filter for 'Wrong Reviews in the last N days", I think I would:

  • Record the maximum number N used by the filter within the last, say, 30 days, and use that as my benchmark for how much data to cache.
  • Upon first use, just fetch all of the data since N days ago, and record that start date with the cached data, along with the date the query was made.
  • Upon subsequent uses, load the cached data, and throw out any datapoints older than N days. Then do an update query of data since the last query, add it to the non-expired data, and save it back into cache (along with the updated start-time and query-time timestamps)

It sounds like more work than it really is, especially with wkof doing the bulk of the fetching and caching.

There’s still the question of whether people would even want to include items that were wrong yesterday but correct today (if N included both days). I don’t think it’s clear what the more common use case would be.

For the reviews endpoint, does it literally return an entry per review done by the user? So can you get more than one result from the API call per item?

Yes. That’s why I didn’t include that in the wkof ItemData. It can be a pretty big number.

I suppose I could add it into ItemData, but require an N parameter so it only caches the last N days. Each item would then have the an array under item.reviews

Still would be a lot of data if they use a ridiculously large N, but it might be useful to add anyway.

Since I’ve been level 60 since before APIv2, I barely have any /reviews data. Would you mind fetching the whole endpoint on your account to see how big it is?

var items;
wkof.Apiv2.fetch_endpoint('reviews').then(function(data){
   items = data;
   var str = JSON.stringify(items);
   console.log('Data is '+str.length+' bytes');
});

It would also help to know what your oldest datapoint is, but I have a rough idea already.

I’ll take a look tonight.

1 Like
16:21:26.818 var items;
wkof.Apiv2.fetch_endpoint('reviews').then(function(data){
   items = data;
   var str = JSON.stringify(items);
   console.log('Data is '+str.length+' bytes');
});
16:21:26.828 Promise {<pending>}
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.096 XHR finished loading: OPTIONS "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:30.744 XHR finished loading: GET "<URL>".
16:21:52.689 Data is 14434898 bytes

14,434898 Megabyte

2 Likes

Data is 12,381,397 bytes.
I had 30,217 entries, for an average of 409.75 bytes per entry.

Here is the first result from the API request:

{"id":8130,"object":"review","url":"https://api.wanikani.com/v2/reviews/8130","data_updated_at":"2017-08-04T14:57:00.943988Z","data":{"created_at":"2017-08-04T14:57:00.943988Z","assignment_id":76837566,"subject_id":4889,"starting_srs_stage":4,"starting_srs_stage_name":"Apprentice IV","ending_srs_stage":5,"ending_srs_stage_name":"Guru I","incorrect_meaning_answers":0,"incorrect_reading_answers":0}}

The first entry is from 2017-08-04. My average level up time has been 14-15 days since shortly before that date, so I think you can assume that a near-max speed person (with a similar error rate) would have roughly twice as much data as me right now. Though obviously that will only continue to go up as users who started after that date get to high levels.

What is the maximum storage size of the database you’re using?

The db is 50MB on most browsers.
I don’t want to push the boundaries, so I think there are two options:

  • Put a cap on the number of days it can cache
  • Compress the data.

I can compress the data about 85%, but I don’t know how long the compression/decompression will take on a large dataset without some tests.

You could try a combination as well if you’re concerned about the performance of the compression/decompression. That way if people just want recent data they don’t have to pay a penalty, but people could get a lot more data if they want.

Do you think a recently quizzed filter would be useful?
If you have it active you would not get the item in the quiz for x hours?