# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

try:
    import jsonresults
    from jsonresults import *
except ImportError:
    print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
    raise

from handlers import master_config

import json
import logging
import unittest

from google.appengine.ext import blobstore
from google.appengine.ext import db
from google.appengine.ext import testbed

FULL_RESULT_EXAMPLE = """ADD_RESULTS({
    "seconds_since_epoch": 1368146629,
    "tests": {
        "media": {
            "encrypted-media": {
                "encrypted-media-v2-events.html": {
                    "bugs": ["crbug.com/1234"],
                    "expected": "TIMEOUT",
                    "actual": "TIMEOUT",
                    "time": 6.0
                },
                "encrypted-media-v2-syntax.html": {
                    "expected": "TIMEOUT",
                    "actual": "TIMEOUT"
                }
            },
            "progress-events-generated-correctly.html": {
                "expected": "PASS FAIL IMAGE TIMEOUT CRASH MISSING",
                "actual": "TIMEOUT",
                "time": 6.0
            },
            "W3C": {
                "audio": {
                    "src": {
                        "src_removal_does_not_trigger_loadstart.html": {
                            "expected": "PASS",
                            "actual": "PASS",
                            "time": 3.5
                        }
                    }
                },
                "video": {
                    "src": {
                        "src_removal_does_not_trigger_loadstart.html": {
                            "expected": "PASS",
                            "actual": "PASS",
                            "time": 1.1
                        },
                        "notrun.html": {
                            "expected": "NOTRUN",
                            "actual": "SKIP",
                            "time": 1.1
                        }
                    }
                }
            },
            "unexpected-skip.html": {
                "expected": "PASS",
                "actual": "SKIP"
            },
            "unexpected-fail.html": {
                "expected": "PASS",
                "actual": "FAIL"
            },
            "flaky-failed.html": {
                "expected": "PASS FAIL",
                "actual": "FAIL"
            },
            "media-document-audio-repaint.html": {
                "expected": "IMAGE",
                "actual": "IMAGE",
                "time": 0.1
            },
            "unexpected-leak.html": {
                "expected": "PASS",
                "actual": "LEAK"
            }
        }
    },
    "skipped": 2,
    "num_regressions": 0,
    "build_number": "3",
    "interrupted": false,
    "layout_tests_dir": "\/tmp\/cr\/src\/third_party\/WebKit\/LayoutTests",
    "version": 3,
    "builder_name": "Webkit",
    "num_passes": 10,
    "pixel_tests_enabled": true,
    "blink_revision": "1234",
    "has_pretty_patch": true,
    "fixable": 25,
    "num_flaky": 0,
    "num_failures_by_type": {
        "CRASH": 3,
        "MISSING": 0,
        "TEXT": 3,
        "IMAGE": 1,
        "PASS": 10,
        "SKIP": 2,
        "TIMEOUT": 16,
        "IMAGE+TEXT": 0,
        "FAIL": 2,
        "AUDIO": 0,
        "LEAK": 1
    },
    "has_wdiff": true,
    "chromium_revision": "5678"
});"""

JSON_RESULTS_OLD_TEMPLATE = (
    '{"[BUILDER_NAME]":{'
    '"allFixableCount":[[TESTDATA_COUNT]],'
    '"blinkRevision":[[TESTDATA_WEBKITREVISION]],'
    '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
    '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
    '"failure_map": %s,'
    '"fixableCount":[[TESTDATA_COUNT]],'
    '"fixableCounts":[[TESTDATA_COUNTS]],'
    '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
    '"tests":{[TESTDATA_TESTS]}'
    '},'
    '"version":[VERSION]'
    '}') % json.dumps(CHAR_TO_FAILURE)

JSON_RESULTS_COUNTS = '{"' + '":[[TESTDATA_COUNT]],"'.join([char for char in CHAR_TO_FAILURE.values()]) + '":[[TESTDATA_COUNT]]}'

JSON_RESULTS_TEMPLATE = (
    '{"[BUILDER_NAME]":{'
    '"blinkRevision":[[TESTDATA_WEBKITREVISION]],'
    '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
    '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
    '"failure_map": %s,'
    '"num_failures_by_type":%s,'
    '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
    '"tests":{[TESTDATA_TESTS]}'
    '},'
    '"version":[VERSION]'
    '}') % (json.dumps(CHAR_TO_FAILURE), JSON_RESULTS_COUNTS)

JSON_RESULTS_COUNTS_TEMPLATE = '{"' + '":[TESTDATA],"'.join([char for char in CHAR_TO_FAILURE]) + '":[TESTDATA]}'

JSON_RESULTS_TEST_LIST_TEMPLATE = '{"Webkit":{"tests":{[TESTDATA_TESTS]}}}'


class MockFile(object):
    @property
    def file_information(self):
        return "master: %s, builder: %s, test_type: %s, build_number: %r, name: %s." % (
            self.master, self.builder, self.test_type, self.build_number, self.name)

    def __init__(self, name='results.json', data=''):
        self.master = 'MockMasterName'
        self.builder = 'MockBuilderName'
        self.test_type = 'MockTestType'
        self.build_number = 0
        self.name = name
        self.data = data

    def save(self, data):
        self.data = data
        return True


class JsonResultsTest(unittest.TestCase):
    def setUp(self):
        self._builder = "Webkit"
        self.old_log_level = logging.root.level
        logging.root.setLevel(logging.ERROR)

    def tearDown(self):
        logging.root.setLevel(self.old_log_level)

    # Use this to get better error messages than just string compare gives.
    def assert_json_equal(self, a, b):
        self.maxDiff = None
        a = json.loads(a) if isinstance(a, str) else a
        b = json.loads(b) if isinstance(b, str) else b
        self.assertEqual(a, b)

    def test_strip_prefix_suffix(self):
        json = "['contents']"
        self.assertEqual(JsonResults._strip_prefix_suffix("ADD_RESULTS(" + json + ");"), json)
        self.assertEqual(JsonResults._strip_prefix_suffix(json), json)

    def _make_test_json(self, test_data, json_string=JSON_RESULTS_TEMPLATE, builder_name="Webkit"):
        if not test_data:
            return ""

        builds = test_data["builds"]
        tests = test_data["tests"]
        if not builds or not tests:
            return ""

        counts = []
        build_numbers = []
        webkit_revision = []
        chrome_revision = []
        times = []
        for build in builds:
            counts.append(JSON_RESULTS_COUNTS_TEMPLATE.replace("[TESTDATA]", build))
            build_numbers.append("1000%s" % build)
            webkit_revision.append("2000%s" % build)
            chrome_revision.append("3000%s" % build)
            times.append("100000%s000" % build)

        json_string = json_string.replace("[BUILDER_NAME]", builder_name)
        json_string = json_string.replace("[TESTDATA_COUNTS]", ",".join(counts))
        json_string = json_string.replace("[TESTDATA_COUNT]", ",".join(builds))
        json_string = json_string.replace("[TESTDATA_BUILDNUMBERS]", ",".join(build_numbers))
        json_string = json_string.replace("[TESTDATA_WEBKITREVISION]", ",".join(webkit_revision))
        json_string = json_string.replace("[TESTDATA_CHROMEREVISION]", ",".join(chrome_revision))
        json_string = json_string.replace("[TESTDATA_TIMES]", ",".join(times))

        version = str(test_data["version"]) if "version" in test_data else "4"
        json_string = json_string.replace("[VERSION]", version)
        json_string = json_string.replace("{[TESTDATA_TESTS]}", json.dumps(tests, separators=(',', ':'), sort_keys=True))
        return json_string

    def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
        aggregated_results = self._make_test_json(aggregated_data, builder_name=self._builder)
        incremental_results = self._make_test_json(incremental_data, builder_name=self._builder)
        incremental_json, _ = JsonResults._get_incremental_json(self._builder, JsonResults._load_json(aggregated_results), is_full_results_format=False)
        merged_results, status_code = JsonResults.merge(self._builder, aggregated_results, JsonResults._load_json(incremental_results), num_runs=max_builds, sort_keys=True)

        if expected_data:
            expected_results = self._make_test_json(expected_data, builder_name=self._builder)
            self.assert_json_equal(merged_results, expected_results)
            self.assertEqual(status_code, 200)
        else:
            self.assertTrue(status_code != 200)

    def _test_get_test_list(self, input_data, expected_data):
        input_results = self._make_test_json(input_data)
        expected_results = JSON_RESULTS_TEST_LIST_TEMPLATE.replace("{[TESTDATA_TESTS]}", json.dumps(expected_data, separators=(',', ':')))
        actual_results = JsonResults.get_test_list(self._builder, input_results)
        self.assert_json_equal(actual_results, expected_results)

    def test_update_files_empty_aggregate_data(self):
        small_file = MockFile(name='results-small.json')
        large_file = MockFile(name='results.json')

        incremental_data = {
            "builds": ["2", "1"],
            "tests": {
                "001.html": {
                    "results": [[200, TEXT]],
                    "times": [[200, 0]],
                }
            }
        }
        incremental_string = self._make_test_json(incremental_data, builder_name=small_file.builder)
        incremental_json = JsonResults._load_json(incremental_string)

        self.assertTrue(JsonResults.update_files(small_file.builder, incremental_json, small_file, large_file, is_full_results_format=False))
        self.assert_json_equal(small_file.data, incremental_string)
        self.assert_json_equal(large_file.data, incremental_string)

    def test_update_files_null_incremental_data(self):
        small_file = MockFile(name='results-small.json')
        large_file = MockFile(name='results.json')

        aggregated_data = {
            "builds": ["2", "1"],
            "tests": {
                "001.html": {
                    "results": [[200, TEXT]],
                    "times": [[200, 0]],
                }
            }
        }
        aggregated_string = self._make_test_json(aggregated_data, builder_name=small_file.builder)

        small_file.data = large_file.data = aggregated_string

        incremental_string = ""

        self.assertEqual(JsonResults.update_files(small_file.builder, incremental_string, small_file, large_file, is_full_results_format=False),
            ('No incremental JSON data to merge.', 403))
        self.assert_json_equal(small_file.data, aggregated_string)
        self.assert_json_equal(large_file.data, aggregated_string)

    def test_update_files_empty_incremental_data(self):
        small_file = MockFile(name='results-small.json')
        large_file = MockFile(name='results.json')

        aggregated_data = {
            "builds": ["2", "1"],
            "tests": {
                "001.html": {
                    "results": [[200, TEXT]],
                    "times": [[200, 0]],
                }
            }
        }
        aggregated_string = self._make_test_json(aggregated_data, builder_name=small_file.builder)

        small_file.data = large_file.data = aggregated_string

        incremental_data = {
            "builds": [],
            "tests": {}
        }
        incremental_string = self._make_test_json(incremental_data, builder_name=small_file.builder)

        self.assertEqual(JsonResults.update_files(small_file.builder, incremental_string, small_file, large_file, is_full_results_format=False),
            ('No incremental JSON data to merge.', 403))
        self.assert_json_equal(small_file.data, aggregated_string)
        self.assert_json_equal(large_file.data, aggregated_string)

    def test_merge_with_empty_aggregated_results(self):
        incremental_data = {
            "builds": ["2", "1"],
            "tests": {
                "001.html": {
                    "results": [[200, TEXT]],
                    "times": [[200, 0]],
                }
            }
        }
        incremental_json = JsonResults._load_json(self._make_test_json(incremental_data))
        incremental_results, _ = JsonResults._get_incremental_json(self._builder, incremental_json, is_full_results_format=False)
        aggregated_results = ""
        merged_results, _ = JsonResults.merge(self._builder, aggregated_results, incremental_results, num_runs=jsonresults.JSON_RESULTS_MAX_BUILDS, sort_keys=True)
        self.assert_json_equal(merged_results, incremental_results)

    def test_failures_by_type_added(self):
        aggregated_results = self._make_test_json({
            "builds": ["2", "1"],
            "tests": {
                "001.html": {
                    "results": [[100, TEXT], [100, FAIL]],
                    "times": [[200, 0]],
                }
            }
        }, json_string=JSON_RESULTS_OLD_TEMPLATE)
        incremental_results = self._make_test_json({
            "builds": ["3"],
            "tests": {
                "001.html": {
                    "results": [[1, TEXT]],
                    "times": [[1, 0]],
                }
            }
        }, json_string=JSON_RESULTS_OLD_TEMPLATE)
        incremental_json, _ = JsonResults._get_incremental_json(self._builder, JsonResults._load_json(incremental_results), is_full_results_format=False)
        merged_results, _ = JsonResults.merge(self._builder, aggregated_results, incremental_json, num_runs=201, sort_keys=True)
        self.assert_json_equal(merged_results, self._make_test_json({
            "builds": ["3", "2", "1"],
            "tests": {
                "001.html": {
                    "results": [[101, TEXT], [100, FAIL]],
                    "times": [[201, 0]],
                }
            }
        }))

    def test_merge_full_results_format(self):
        expected_incremental_results = {
            "Webkit": {
                "blinkRevision": ["1234"],
                "buildNumbers": ["3"],
                "chromeRevision": ["5678"],
                "failure_map": CHAR_TO_FAILURE,
                "num_failures_by_type": {"AUDIO": [0], "CRASH": [3], "FAIL": [2], "IMAGE": [1], "IMAGE+TEXT": [0], "MISSING": [0], "PASS": [10], "SKIP": [2], "TEXT": [3], "TIMEOUT": [16], "LEAK": [1]},
                "secondsSinceEpoch": [1368146629],
                "tests": {
                    "media": {
                        "W3C": {
                            "audio": {
                                "src": {
                                    "src_removal_does_not_trigger_loadstart.html": {
                                        "results": [[1, PASS]],
                                        "times": [[1, 4]],
                                    }
                                }
                            }
                        },
                        "encrypted-media": {
                            "encrypted-media-v2-events.html": {
                                "bugs": ["crbug.com/1234"],
                                "expected": "TIMEOUT",
                                "results": [[1, TIMEOUT]],
                                "times": [[1, 6]],
                            },
                            "encrypted-media-v2-syntax.html": {
                                "expected": "TIMEOUT",
                                "results": [[1, TIMEOUT]],
                                "times": [[1, 0]],
                            }
                        },
                        "media-document-audio-repaint.html": {
                            "expected": "IMAGE",
                            "results": [[1, IMAGE]],
                            "times": [[1, 0]],
                        },
                        "progress-events-generated-correctly.html": {
                            "expected": "PASS FAIL IMAGE TIMEOUT CRASH MISSING",
                            "results": [[1, TIMEOUT]],
                            "times": [[1, 6]],
                        },
                        "flaky-failed.html": {
                            "expected": "PASS FAIL",
                            "results": [[1, FAIL]],
                            "times": [[1, 0]],
                        },
                        "unexpected-fail.html": {
                            "results": [[1, FAIL]],
                            "times": [[1, 0]],
                        },
                        "unexpected-leak.html": {
                            "results": [[1, LEAK]],
                            "times": [[1, 0]],
                        },
                    }
                }
            },
            "version": 4
        }

        aggregated_results = ""
        incremental_json, _ = JsonResults._get_incremental_json(self._builder, JsonResults._load_json(FULL_RESULT_EXAMPLE), is_full_results_format=True)
        merged_results, _ = JsonResults.merge("Webkit", aggregated_results, incremental_json, num_runs=jsonresults.JSON_RESULTS_MAX_BUILDS, sort_keys=True)
        self.assert_json_equal(merged_results, expected_incremental_results)

    def test_merge_empty_aggregated_results(self):
        # No existing aggregated results.
        # Merged results == new incremental results.
        self._test_merge(
            # Aggregated results
            None,
            # Incremental results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]}}},
            # Expected result
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]}}})

    def test_merge_duplicate_build_number(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[100, TEXT]],
                           "times": [[100, 0]]}}},
            # Incremental results
            {"builds": ["2"],
             "tests": {"001.html": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]}}},
            # Expected results
            None)

    def test_merge_incremental_single_test_single_run_same_result(self):
        # Incremental results has the latest build and same test results for
        # that run.
        # Insert the incremental results at the first place and sum number
        # of runs for TEXT (200 + 1) to get merged results.
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[201, TEXT]],
                           "times": [[201, 0]]}}})

    def test_merge_single_test_single_run_different_result(self):
        # Incremental results has the latest build but different test results
        # for that run.
        # Insert the incremental results at the first place.
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, IMAGE]],
                           "times": [[1, 1]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[1, IMAGE], [200, TEXT]],
                           "times": [[1, 1], [200, 0]]}}})

    def test_merge_single_test_single_run_result_changed(self):
        # Incremental results has the latest build but results which differ from
        # the latest result (but are the same as an older result).
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT], [10, IMAGE]],
                           "times": [[200, 0], [10, 1]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, IMAGE]],
                           "times": [[1, 1]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[1, IMAGE], [200, TEXT], [10, IMAGE]],
                           "times": [[1, 1], [200, 0], [10, 1]]}}})

    def test_merge_multiple_tests_single_run(self):
        # All tests have incremental updates.
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]},
                       "002.html": {
                           "results": [[100, IMAGE]],
                           "times": [[100, 1]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]},
                       "002.html": {
                           "results": [[1, IMAGE]],
                           "times": [[1, 1]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[201, TEXT]],
                           "times": [[201, 0]]},
                       "002.html": {
                           "results": [[101, IMAGE]],
                           "times": [[101, 1]]}}})

    def test_merge_multiple_tests_single_run_one_no_result(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]},
                       "002.html": {
                           "results": [[100, IMAGE]],
                           "times": [[100, 1]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"002.html": {
                           "results": [[1, IMAGE]],
                           "times": [[1, 1]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[1, NO_DATA], [200, TEXT]],
                           "times": [[201, 0]]},
                       "002.html": {
                           "results": [[101, IMAGE]],
                           "times": [[101, 1]]}}})

    def test_merge_single_test_multiple_runs(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]}}},
            # Incremental results
            {"builds": ["4", "3"],
             "tests": {"001.html": {
                           "results": [[2, IMAGE], [1, FAIL]],
                           "times": [[3, 2]]}}},
            # Expected results
            {"builds": ["4", "3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[1, FAIL], [2, IMAGE], [200, TEXT]],
                           "times": [[3, 2], [200, 0]]}}})

    def test_merge_multiple_tests_multiple_runs(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]},
                       "002.html": {
                           "results": [[10, IMAGE_PLUS_TEXT]],
                           "times": [[10, 0]]}}},
            # Incremental results
            {"builds": ["4", "3"],
             "tests": {"001.html": {
                           "results": [[2, IMAGE]],
                           "times": [[2, 2]]},
                       "002.html": {
                           "results": [[1, CRASH]],
                           "times": [[1, 1]]}}},
            # Expected results
            {"builds": ["4", "3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[2, IMAGE], [200, TEXT]],
                           "times": [[2, 2], [200, 0]]},
                       "002.html": {
                           "results": [[1, CRASH], [10, IMAGE_PLUS_TEXT]],
                           "times": [[1, 1], [10, 0]]}}})

    def test_merge_incremental_result_older_build(self):
        # Test the build in incremental results is older than the most recent
        # build in aggregated results.
        self._test_merge(
            # Aggregated results
            {"builds": ["3", "1"],
             "tests": {"001.html": {
                           "results": [[5, TEXT]],
                           "times": [[5, 0]]}}},
            # Incremental results
            {"builds": ["2"],
             "tests": {"001.html": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]}}},
            # Expected no merge happens.
            {"builds": ["2", "3", "1"],
             "tests": {"001.html": {
                           "results": [[6, TEXT]],
                           "times": [[6, 0]]}}})

    def test_merge_incremental_result_same_build(self):
        # Test the build in incremental results is same as the build in
        # aggregated results.
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[5, TEXT]],
                           "times": [[5, 0]]}}},
            # Incremental results
            {"builds": ["3", "2"],
             "tests": {"001.html": {
                           "results": [[2, TEXT]],
                           "times": [[2, 0]]}}},
            # Expected no merge happens.
            {"builds": ["3", "2", "2", "1"],
             "tests": {"001.html": {
                           "results": [[7, TEXT]],
                           "times": [[7, 0]]}}})

    def test_merge_remove_new_test(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[199, TEXT]],
                           "times": [[199, 0]]},
                       }},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]},
                       "002.html": {
                           "results": [[1, PASS]],
                           "times": [[1, 0]]},
                       "notrun.html": {
                           "results": [[1, NOTRUN]],
                           "times": [[1, 0]]},
                       "003.html": {
                           "results": [[1, NO_DATA]],
                           "times": [[1, 0]]},
                        }},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[200, TEXT]],
                           "times": [[200, 0]]},
                       }},
            max_builds=200)

    def test_merge_remove_test(self):
        self._test_merge(
            # Aggregated results
            {
                "builds": ["2", "1"],
                "tests": {
                    "directory": {
                        "directory": {
                            "001.html": {
                                "results": [[200, PASS]],
                                "times": [[200, 0]]
                            }
                        }
                    },
                    "002.html": {
                        "results": [[10, TEXT]],
                        "times": [[10, 0]]
                    },
                    "003.html": {
                        "results": [[190, PASS], [9, NO_DATA], [1, TEXT]],
                        "times": [[200, 0]]
                    },
                }
            },
            # Incremental results
            {
                "builds": ["3"],
                "tests": {
                    "directory": {
                        "directory": {
                            "001.html": {
                                "results": [[1, PASS]],
                                "times": [[1, 0]]
                            }
                        }
                    },
                    "002.html": {
                        "results": [[1, PASS]],
                        "times": [[1, 0]]
                    },
                    "003.html": {
                        "results": [[1, PASS]],
                        "times": [[1, 0]]
                    },
                }
            },
            # Expected results
            {
                "builds": ["3", "2", "1"],
                "tests": {
                    "002.html": {
                        "results": [[1, PASS], [10, TEXT]],
                        "times": [[11, 0]]
                    }
                }
            },
            max_builds=200)

    def test_merge_updates_expected(self):
        self._test_merge(
            # Aggregated results
            {
                "builds": ["2", "1"],
                "tests": {
                    "directory": {
                        "directory": {
                            "001.html": {
                                "expected": "FAIL",
                                "results": [[200, PASS]],
                                "times": [[200, 0]]
                            }
                        }
                    },
                    "002.html": {
                        "bugs": ["crbug.com/1234"],
                        "expected": "FAIL",
                        "results": [[10, TEXT]],
                        "times": [[10, 0]]
                    },
                    "003.html": {
                        "expected": "FAIL",
                        "results": [[190, PASS], [9, NO_DATA], [1, TEXT]],
                        "times": [[200, 0]]
                    },
                    "004.html": {
                        "results": [[199, PASS], [1, TEXT]],
                        "times": [[200, 0]]
                    },
                }
            },
            # Incremental results
            {
                "builds": ["3"],
                "tests": {
                    "002.html": {
                        "expected": "PASS",
                        "results": [[1, PASS]],
                        "times": [[1, 0]]
                    },
                    "003.html": {
                        "expected": "TIMEOUT",
                        "results": [[1, PASS]],
                        "times": [[1, 0]]
                    },
                    "004.html": {
                        "bugs": ["crbug.com/1234"],
                        "results": [[1, PASS]],
                        "times": [[1, 0]]
                    },
                }
            },
            # Expected results
            {
                "builds": ["3", "2", "1"],
                "tests": {
                    "002.html": {
                        "results": [[1, PASS], [10, TEXT]],
                        "times": [[11, 0]]
                    },
                    "003.html": {
                        "expected": "TIMEOUT",
                        "results": [[191, PASS], [9, NO_DATA]],
                        "times": [[200, 0]]
                    },
                    "004.html": {
                        "bugs": ["crbug.com/1234"],
                        "results": [[200, PASS]],
                        "times": [[200, 0]]
                    },
                }
            },
            max_builds=200)


    def test_merge_keep_test_with_all_pass_but_slow_time(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, PASS]],
                           "times": [[200, jsonresults.JSON_RESULTS_MIN_TIME]]},
                       "002.html": {
                           "results": [[10, TEXT]],
                           "times": [[10, 0]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, PASS]],
                           "times": [[1, 1]]},
                       "002.html": {
                           "results": [[1, PASS]],
                           "times": [[1, 0]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[201, PASS]],
                           "times": [[1, 1], [200, jsonresults.JSON_RESULTS_MIN_TIME]]},
                       "002.html": {
                           "results": [[1, PASS], [10, TEXT]],
                           "times": [[11, 0]]}}})

    def test_merge_pruning_slow_tests_for_debug_builders(self):
        self._builder = "MockBuilder(dbg)"
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[200, PASS]],
                           "times": [[200, 3 * jsonresults.JSON_RESULTS_MIN_TIME]]},
                       "002.html": {
                           "results": [[10, TEXT]],
                           "times": [[10, 0]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, PASS]],
                           "times": [[1, 1]]},
                       "002.html": {
                           "results": [[1, PASS]],
                           "times": [[1, 0]]},
                       "003.html": {
                           "results": [[1, PASS]],
                           "times": [[1, jsonresults.JSON_RESULTS_MIN_TIME]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[201, PASS]],
                           "times": [[1, 1], [200, 3 * jsonresults.JSON_RESULTS_MIN_TIME]]},
                       "002.html": {
                           "results": [[1, PASS], [10, TEXT]],
                           "times": [[11, 0]]}}})

    def test_merge_prune_extra_results(self):
        # Remove items from test results and times that exceed the max number
        # of builds to track.
        max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[max_builds, TEXT], [1, IMAGE]],
                           "times": [[max_builds, 0], [1, 1]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, TIMEOUT]],
                           "times": [[1, 1]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[1, TIMEOUT], [max_builds, TEXT]],
                           "times": [[1, 1], [max_builds, 0]]}}})

    def test_merge_prune_extra_results_small(self):
        # Remove items from test results and times that exceed the max number
        # of builds to track, using smaller threshold.
        max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[max_builds, TEXT], [1, IMAGE]],
                           "times": [[max_builds, 0], [1, 1]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, TIMEOUT]],
                           "times": [[1, 1]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[1, TIMEOUT], [max_builds, TEXT]],
                           "times": [[1, 1], [max_builds, 0]]}}},
            int(max_builds))

    def test_merge_prune_extra_results_with_new_result_of_same_type(self):
        # Test that merging in a new result of the same type as the last result
        # causes old results to fall off.
        max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"001.html": {
                           "results": [[max_builds, TEXT], [1, NO_DATA]],
                           "times": [[max_builds, 0], [1, 1]]}}},
            # Incremental results
            {"builds": ["3"],
             "tests": {"001.html": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]}}},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"001.html": {
                           "results": [[max_builds, TEXT]],
                           "times": [[max_builds, 0]]}}},
            int(max_builds))

    def test_merge_build_directory_hierarchy(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"bar": {"baz": {
                           "003.html": {
                                "results": [[25, TEXT]],
                                "times": [[25, 0]]}}},
                       "foo": {
                           "001.html": {
                                "results": [[50, TEXT]],
                                "times": [[50, 0]]},
                           "002.html": {
                                "results": [[100, IMAGE]],
                                "times": [[100, 0]]}}},
              "version": 4},
            # Incremental results
            {"builds": ["3"],
             "tests": {"baz": {
                           "004.html": {
                               "results": [[1, IMAGE]],
                               "times": [[1, 0]]}},
                       "foo": {
                           "001.html": {
                               "results": [[1, TEXT]],
                               "times": [[1, 0]]},
                           "002.html": {
                               "results": [[1, IMAGE]],
                               "times": [[1, 0]]}}},
             "version": 4},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"bar": {"baz": {
                           "003.html": {
                               "results": [[1, NO_DATA], [25, TEXT]],
                               "times": [[26, 0]]}}},
                       "baz": {
                           "004.html": {
                               "results": [[1, IMAGE]],
                               "times": [[1, 0]]}},
                       "foo": {
                           "001.html": {
                               "results": [[51, TEXT]],
                               "times": [[51, 0]]},
                           "002.html": {
                               "results": [[101, IMAGE]],
                               "times": [[101, 0]]}}},
             "version": 4})

    # FIXME(aboxhall): Add some tests for xhtml/svg test results.

    def test_get_test_name_list(self):
        # Get test name list only. Don't include non-test-list data and
        # of test result details.
        # FIXME: This also tests a temporary bug in the data where directory-level
        # results have a results and times values. Once that bug is fixed,
        # remove this test-case and assert we don't ever hit it.
        self._test_get_test_list(
            # Input results
            {"builds": ["3", "2", "1"],
             "tests": {"foo": {
                           "001.html": {
                               "results": [[200, PASS]],
                               "times": [[200, 0]]},
                           "results": [[1, NO_DATA]],
                           "times": [[1, 0]]},
                       "002.html": {
                           "results": [[10, TEXT]],
                           "times": [[10, 0]]}}},
            # Expected results
            {"foo": {"001.html": {}}, "002.html": {}})

    def test_gtest(self):
        self._test_merge(
            # Aggregated results
            {"builds": ["2", "1"],
             "tests": {"foo.bar": {
                           "results": [[50, TEXT]],
                           "times": [[50, 0]]},
                       "foo.bar2": {
                           "results": [[100, IMAGE]],
                           "times": [[100, 0]]},
                       "test.failed": {
                           "results": [[5, FAIL]],
                           "times": [[5, 0]]},
                       },
             "version": 3},
            # Incremental results
            {"builds": ["3"],
             "tests": {"foo.bar2": {
                           "results": [[1, IMAGE]],
                           "times": [[1, 0]]},
                       "foo.bar3": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]},
                       "test.failed": {
                           "results": [[5, FAIL]],
                           "times": [[5, 0]]},
                       },
             "version": 4},
            # Expected results
            {"builds": ["3", "2", "1"],
             "tests": {"foo.bar": {
                           "results": [[1, NO_DATA], [50, TEXT]],
                           "times": [[51, 0]]},
                       "foo.bar2": {
                           "results": [[101, IMAGE]],
                           "times": [[101, 0]]},
                       "foo.bar3": {
                           "results": [[1, TEXT]],
                           "times": [[1, 0]]},
                       "test.failed": {
                           "results": [[10, FAIL]],
                           "times": [[10, 0]]},
                       },
             "version": 4})

    def test_deprecated_master_name(self):
        tb = testbed.Testbed()
        tb.activate()
        tb.init_datastore_v3_stub()
        tb.init_blobstore_stub()

        master = master_config.getMaster('chromium.chromiumos')
        builder = 'test-builder'
        test_type = 'test-type'

        test_data = [
            {
                'tests': {
                    'Test1.testproc1': {
                        'expected': 'PASS',
                        'actual': 'PASS',
                        'time': 1,
                    }
                },
                'build_number': '123',
                'version': JSON_RESULTS_HIERARCHICAL_VERSION,
                'builder_name': builder,
                'blink_revision': '12345',
                'seconds_since_epoch': 1406123456,
                'num_failures_by_type': {
                    'FAIL': 0,
                    'SKIP': 0,
                    'PASS': 1
                },
                'chromium_revision': '67890',
            },
            {
                'tests': {
                    'Test2.testproc2': {
                        'expected': 'PASS',
                        'actual': 'FAIL',
                        'time': 2,
                    }
                },
                'build_number': '456',
                'version': JSON_RESULTS_HIERARCHICAL_VERSION,
                'builder_name': builder,
                'blink_revision': '54321',
                'seconds_since_epoch': 1406654321,
                'num_failures_by_type': {
                    'FAIL': 1,
                    'SKIP': 0,
                    'PASS': 0
                },
                'chromium_revision': '98765',
            },
        ]

        # Upload a file using old master name

        # Seed results files using the old name.
        JsonResults.update(master['name'], builder, test_type, test_data[0], None, True)
        # Update results files using the new name.
        JsonResults.update(master['url_name'], builder, test_type, test_data[1], master['name'], True)
        # Verify that the file keyed by url_name contains both sets of results.
        files = TestFile.get_files(master['url_name'], builder, test_type, None, None, limit=3)
        self.assertEqual(len(files), 2)
        for f in files:
            j = json.loads(f.data)
            self.assertItemsEqual(j[builder]['blinkRevision'], ['12345', '54321'])

        tb.deactivate()

    def test_normalize_results_with_top_level_results_key_does_not_crash(self):
        aggregated_json = {
            'Linux Tests': {
                'results': {'foo': {'results': [(1, 'P')],
                                    'times': [(1, 1)]}},
            }
        }
        JsonResults._normalize_results(aggregated_json, 1, 2)

if __name__ == '__main__':
    unittest.main()
