model.py

# -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2013-2017 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""A local Predictive Model.

This module defines a Model to make predictions locally or
embedded into your application without needing to send requests to
BigML.io.

This module cannot only save you a few credits, but also enormously
reduce the latency for each prediction and let you use your models
offline.

You can also visualize your predictive model in IF-THEN rule format
and even generate a python function that implements the model.

Example usage (assuming that you have previously set up the BIGML_USERNAME
and BIGML_API_KEY environment variables and that you own the model/id below):

from bigml.api import BigML
from bigml.model import Model

api = BigML()

model = Model('model/5026965515526876630001b2')
model.predict({"petal length": 3, "petal width": 1})

You can also see model in a IF-THEN rule format with:

model.rules()

Or auto-generate a python function code for the model with:

model.python()

"""
import logging
import sys
import locale
import json


from functools import partial

from bigml.api import FINISHED
from bigml.api import (BigML, get_model_id, get_status)
from bigml.util import (slugify, markdown_cleanup,
                        prefix_as_comment, utf8,
                        find_locale, cast)
from bigml.util import DEFAULT_LOCALE
from bigml.tree import Tree, LAST_PREDICTION, PROPORTIONAL
from bigml.boostedtree import BoostedTree
from bigml.predicate import Predicate
from bigml.basemodel import BaseModel, retrieve_resource, print_importance
from bigml.basemodel import ONLY_MODEL, EXCLUDE_FIELDS
from bigml.modelfields import check_model_fields
from bigml.multivote import ws_confidence
from bigml.io import UnicodeWriter
from bigml.path import Path, BRIEF
from bigml.prediction import Prediction


LOGGER = logging.getLogger('BigML')

# we use the atof conversion for integers to include integers written as
# 10.0
PYTHON_CONV = {
    "double": "locale.atof",
    "float": "locale.atof",
    "integer": "lambda x: int(locale.atof(x))",
    "int8": "lambda x: int(locale.atof(x))",
    "int16": "lambda x: int(locale.atof(x))",
    "int32": "lambda x: int(locale.atof(x))",
    "int64": "lambda x: long(locale.atof(x))",
    "day": "lambda x: int(locale.atof(x))",
    "month": "lambda x: int(locale.atof(x))",
    "year": "lambda x: int(locale.atof(x))",
    "hour": "lambda x: int(locale.atof(x))",
    "minute": "lambda x: int(locale.atof(x))",
    "second": "lambda x: int(locale.atof(x))",
    "millisecond": "lambda x: int(locale.atof(x))",
    "day-of-week": "lambda x: int(locale.atof(x))",
    "day-of-month": "lambda x: int(locale.atof(x))"
}

PYTHON_FUNC = dict([(numtype, eval(function))
                    for numtype, function in PYTHON_CONV.iteritems()])

INDENT = u'    '

STORAGE = './storage'

DEFAULT_IMPURITY = 0.2

OPERATING_POINT_KINDS = ["probability", "confidence"]


def print_distribution(distribution, out=sys.stdout):
    """Prints distribution data

    """
    total = reduce(lambda x, y: x + y,
                   [group[1] for group in distribution])
    for group in distribution:
        out.write(utf8(
            u"    %s: %.2f%% (%d instance%s)\n" % (
                group[0],
                round(group[1] * 1.0 / total, 4) * 100,
                group[1],
                u"" if group[1] == 1 else u"s")))


def parse_operating_point(operating_point, operating_kinds, class_names):
    """Checks the operating point contents and extracts the three defined
    variables

    """
    if "kind" not in operating_point:
        raise ValueError("Failed to find the kind of operating point.")
    elif operating_point["kind"] not in operating_kinds:
        raise ValueError("Unexpected operating point kind. Allowed values"
                         " are: %s." % ", ".join(operating_kinds))
    if "threshold" not in operating_point:
        raise ValueError("Failed to find the threshold of the operating"
                         "point.")
    if operating_point["threshold"] > 1 or \
            operating_point["threshold"] < 0:
        raise ValueError("The threshold value should be in the 0 to 1"
                         " range.")
    if "positive_class" not in operating_point:
        raise ValueError("The operating point needs to have a"
                         " positive_class attribute.")
    else:
        positive_class = operating_point["positive_class"]
        if positive_class not in class_names:
            raise ValueError("The positive class must be one of the"
                             "objective field classes: %s." %
                             ", ".join(class_names))
    kind = operating_point["kind"]
    threshold = operating_point["threshold"]

    return kind, threshold, positive_class


class Model(BaseModel):
    """ A lightweight wrapper around a Tree model.

    Uses a BigML remote model to build a local version that can be used
    to generate predictions locally.

    """

    def __init__(self, model, api=None, fields=None):
        """The Model constructor can be given as first argument:
            - a model structure
            - a model id
            - a path to a JSON file containing a model structure

        """
        self.resource_id = None
        self.ids_map = {}
        self.terms = {}
        self.regression = False
        self.boosting = None
        self.class_names = None
        # the string can be a path to a JSON file
        if isinstance(model, basestring):
            try:
                with open(model) as model_file:
                    model = json.load(model_file)
                    self.resource_id = get_model_id(model)
                    if self.resource_id is None:
                        raise ValueError("The JSON file does not seem"
                                         " to contain a valid BigML model"
                                         " representation.")
            except IOError:
                # if it is not a path, it can be a model id
                self.resource_id = get_model_id(model)
                if self.resource_id is None:
                    if model.find('model/') > -1:
                        raise Exception(
                            api.error_message(model,
                                              resource_type='model',
                                              method='get'))
                    else:
                        raise IOError("Failed to open the expected JSON file"
                                      " at %s" % model)
            except ValueError:
                raise ValueError("Failed to interpret %s."
                                 " JSON file expected.")

        # checks whether the information needed for local predictions is in
        # the first argument
        if isinstance(model, dict) and \
                not fields and \
                not check_model_fields(model):
            # if the fields used by the model are not
            # available, use only ID to retrieve it again
            model = get_model_id(model)
            self.resource_id = model

        if not (isinstance(model, dict) and 'resource' in model and
                model['resource'] is not None):
            if api is None:
                api = BigML(storage=STORAGE)
            if fields is not None and isinstance(fields, dict):
                query_string = EXCLUDE_FIELDS
            else:
                query_string = ONLY_MODEL
            model = retrieve_resource(api, self.resource_id,
                                      query_string=query_string,
                                      no_check_fields=fields is not None)
        else:
            self.resource_id = get_model_id(model)
        BaseModel.__init__(self, model, api=api, fields=fields)
        if 'object' in model and isinstance(model['object'], dict):
            model = model['object']

        if 'model' in model and isinstance(model['model'], dict):
            status = get_status(model)
            if 'code' in status and status['code'] == FINISHED:

                # boosting models are to be handled using the BoostedTree
                # class
                if model.get("boosted_ensemble"):
                    self.boosting = model.get('boosting', False)
                if self.boosting == {}:
                    self.boosting = False

                self.regression = \
                    not self.boosting and \
                    self.fields[self.objective_id]['optype'] == 'numeric' \
                    or (self.boosting and \
                    self.boosting.get("objective_class") is None)
                if not hasattr(self, 'tree_class'):
                    self.tree_class = Tree if not self.boosting else \
                        BoostedTree

                if self.boosting:
                    self.tree = self.tree_class(
                        model['model']['root'],
                        self.fields,
                        objective_field=self.objective_id)
                else:
                    distribution = model['model']['distribution']['training']
                    # will store global information in the tree: regression and
                    # max_bins number
                    tree_info = {'max_bins': 0}
                    self.tree = self.tree_class(
                        model['model']['root'],
                        self.fields,
                        objective_field=self.objective_id,
                        root_distribution=distribution,
                        parent_id=None,
                        ids_map=self.ids_map,
                        tree_info=tree_info)

                    self.tree.regression = tree_info['regression']

                    if self.tree.regression:
                        try:
                            import numpy
                            import scipy
                            self._max_bins = tree_info['max_bins']
                            self.regression_ready = True
                        except ImportError:
                            self.regression_ready = False
                    else:
                        root_dist = self.tree.distribution
                        self.class_names = sorted([category[0]
                                                   for category in root_dist])
            else:
                raise Exception("The model isn't finished yet")
        else:
            raise Exception("Cannot create the Model instance. Could not"
                            " find the 'model' key in the resource:\n\n%s" %
                            model)

    def list_fields(self, out=sys.stdout):
        """Prints descriptions of the fields for this model.

        """
        self.tree.list_fields(out)

    def get_leaves(self, filter_function=None):
        """Returns a list that includes all the leaves of the model.

           filter_function should be a function that returns a boolean
           when applied to each leaf node.
        """
        return self.tree.get_leaves(filter_function=filter_function)

    def impure_leaves(self, impurity_threshold=DEFAULT_IMPURITY):
        """Returns a list of leaves that are impure

        """
        if self.regression or self.boosting:
            raise AttributeError("This method is available for non-boosting"
                                 " categorization models only.")
        def is_impure(node, impurity_threshold=impurity_threshold):
            """Returns True if the gini impurity of the node distribution
               goes above the impurity threshold.

            """
            return node.get('impurity') > impurity_threshold

        is_impure = partial(is_impure, impurity_threshold=impurity_threshold)
        return self.get_leaves(filter_function=is_impure)

    def _to_output(self, output_map, compact, value_key):
        if compact:
            return [output_map.get(name, 0.0) for name in self.class_names]
        else:
            output = []
            for name in self.class_names:
                output.append({
                    'category': name,
                    value_key: output_map.get(name, 0.0)
                })
            return output

    def predict_confidence(self, input_data, by_name=True,
                           missing_strategy=LAST_PREDICTION,
                           compact=False):
        """For classification models, Predicts a one-vs.-rest confidence value
        for each possible output class, based on input values.  This
        confidence value is a lower confidence bound on the predicted
        probability of the given class.  The input fields must be a
        dictionary keyed by field name for field ID.

        For regressions, the output is a single element list
        containing the prediction.

        :param input_data: Input data to be predicted
        :param by_name: Boolean that is set to True if field_names (as
                        alternative to field ids) are used in the
                        input_data dict
        :param missing_strategy: LAST_PREDICTION|PROPORTIONAL missing strategy
                                 for missing fields
        :param compact: If False, prediction is returned as a list of maps, one
                        per class, with the keys "prediction" and "confidence"
                        mapped to the name of the class and its confidence,
                        respectively.  If True, returns a list of confidences
                        ordered by the sorted order of the class names.

        """
        if self.regression or self.boosting:
            raise AttributeError("This method is available for non-boosting"
                                 " categorization models only.")

        root_dist = self.tree.distribution
        category_map = {category[0]: 0.0 for category in root_dist}
        prediction = self.predict(input_data,
                                  by_name=by_name,
                                  missing_strategy=missing_strategy,
                                  add_distribution=True)

        distribution = prediction['distribution']

        for class_info in distribution:
            name = class_info[0]
            category_map[name] = ws_confidence(name, distribution)

        return self._to_output(category_map, compact, "confidence")

    def predict_probability(self, input_data, by_name=True,
                            missing_strategy=LAST_PREDICTION,
                            compact=False):
        """For classification models, Predicts a probability for
        each possible output class, based on input values.  The input
        fields must be a dictionary keyed by field name for field ID.

        For regressions, the output is a single element list
        containing the prediction.

        :param input_data: Input data to be predicted
        :param by_name: Boolean that is set to True if field_names (as
                        alternative to field ids) are used in the
                        input_data dict
        :param missing_strategy: LAST_PREDICTION|PROPORTIONAL missing strategy
                                 for missing fields
        :param compact: If False, prediction is returned as a list of maps, one
                        per class, with the keys "prediction" and "probability"
                        mapped to the name of the class and it's probability,
                        respectively.  If True, returns a list of probabilities
                        ordered by the sorted order of the class names.
        """
        if self.regression:
            prediction = self.predict(input_data,
                                      by_name=by_name,
                                      missing_strategy=missing_strategy)

            if compact:
                output = [prediction]
            else:
                output = {'prediction': prediction}
        else:
            root_dist = self.tree.distribution

            if self.tree.weighted:
                category_map = {category[0]: 0.0 for category in root_dist}
                instances = 0.0
            else:
                total = float(sum([category[1] for category in root_dist]))
                category_map = {category[0]: category[1] / total
                                for category in root_dist}
                instances = 1.0

            prediction = self.predict(input_data,
                                      by_name=by_name,
                                      missing_strategy=missing_strategy,
                                      add_distribution=True)

            distribution = prediction['distribution']

            for class_info in distribution:
                category_map[class_info[0]] += class_info[1]
                instances += class_info[1]

            for k in category_map:
                category_map[k] /= instances

            output = self._to_output(category_map, compact, "probability")

        return output

    def predict_operating(self, input_data, by_name=True,
                          missing_strategy=LAST_PREDICTION,
                          operating_point=None, compact=False):
        """Computes the prediction based on a user-given operating point.

        """

        kind, threshold, positive_class = parse_operating_point( \
            operating_point, OPERATING_POINT_KINDS, self.class_names)
        if kind == "probability":
            predictions = self.predict_probability(input_data, by_name,
                                                   missing_strategy, compact)
        else:
            predictions = self.predict_confidence(input_data, by_name,
                                                  missing_strategy, compact)

        position = self.class_names.index(positive_class)
        if predictions[position][kind] < threshold:
            # if the threshold is not met, the alternative class with
            # highest probability or confidence is returned
            prediction = sorted(predictions,
                                key=lambda x: - x[kind])[0 : 2]
            if prediction[0]["category"] == positive_class:
                prediction = prediction[1]
            else:
                prediction = prediction[0]
        else:
            prediction = predictions[position]
        prediction["prediction"] = prediction["category"]
        del prediction["category"]
        return prediction

    def predict(self, input_data, by_name=True,
                print_path=False, out=sys.stdout, with_confidence=False,
                missing_strategy=LAST_PREDICTION,
                add_confidence=False,
                add_path=False,
                add_distribution=False,
                add_count=False,
                add_median=False,
                add_next=False,
                add_min=False,
                add_max=False,
                add_unused_fields=False,
                add_probability=False,
                operating_point=None):
        """Makes a prediction based on a number of field values.

        By default the input fields must be keyed by field name but you can use
        `by_name=False` to input them directly keyed by id.

        input_data: Input data to be predicted
        by_name: Boolean, True if input_data is keyed by names
        print_path: Boolean, if True the rules that lead to the prediction
                    are printed
        out: output handler
        with_confidence: Boolean, if True, all the information in the node
                         (prediction, confidence, distribution and count)
                         is returned in a list format
        missing_strategy: LAST_PREDICTION|PROPORTIONAL missing strategy for
                          missing fields
        add_confidence: Boolean, if True adds confidence (or probability)
                        to the dict output
        add_path: Boolean, if True adds path to the dict output
        add_distribution: Boolean, if True adds distribution info to the
                          dict output
        add_count: Boolean, if True adds the number of instances in the
                       node to the dict output
        add_median: Boolean, if True adds the median of the values in
                    the distribution
        add_next: Boolean, if True adds the field that determines next
                  split in the tree
        add_min: Boolean, if True adds the minimum value in the prediction's
                 distribution (for regressions only)
        add_max: Boolean, if True adds the maximum value in the prediction's
                 distribution (for regressions only)
        add_unused_fields: Boolean, if True adds the information about the
                           fields in the input_data that are not being used
                           in the model as predictors.
        add_probability: Boolean, if True adds probability
                         to the dict output (only if operating_point is used)
        operating_point: In classification models, this is the point of the
                         ROC curve where the model will be used at. The
                         operating point can be defined in terms of:
                         - the positive_class, the class that is important to
                           predict accurately
                         - the probability_threshold (or confidence_threshold),
                           the probability (or confidence) that is stablished
                           as minimum for the positive_class to be predicted.
                         The operating_point is then defined as a map with
                         two attributes, e.g.:
                           {"positive_class": "Iris-setosa",
                            "probability_threshold": 0.5}
                         or
                           {"positive_class": "Iris-setosa",
                            "confidence_threshold": 0.5}
        """

        # Checks and cleans input_data leaving the fields used in the model
        unused_fields = []
        new_data = self.filter_input_data( \
            input_data, by_name=by_name,
            add_unused_fields=add_unused_fields)
        if add_unused_fields:
            input_data, unused_fields = new_data
        else:
            input_data = new_data

        # Strips affixes for numeric values and casts to the final field type
        cast(input_data, self.fields)

        return self._predict( \
            input_data, by_name=by_name,
            print_path=print_path,
            out=out,
            with_confidence=with_confidence,
            missing_strategy=missing_strategy,
            add_confidence=add_confidence,
            add_path=add_path,
            add_distribution=add_distribution,
            add_count=add_count,
            add_median=add_median,
            add_next=add_next,
            add_min=add_min,
            add_max=add_max,
            add_unused_fields=add_unused_fields,
            add_probability=add_probability,
            operating_point=operating_point,
            unused_fields=unused_fields)

    def _predict(self, input_data, by_name=True,
                 print_path=False, out=sys.stdout, with_confidence=False,
                 missing_strategy=LAST_PREDICTION,
                 add_confidence=False,
                 add_path=False,
                 add_distribution=False,
                 add_count=False,
                 add_median=False,
                 add_next=False,
                 add_min=False,
                 add_max=False,
                 add_unused_fields=False,
                 add_probability=False,
                 operating_point=None,
                 unused_fields=None):
        """Makes a prediction based on a number of field values. Please,
        note that this function does not check the types for the input
        provided, so it's unsafe to use it directly without prior checking.

        By default the input fields must be keyed by field name but you can use
        `by_name=False` to input them directly keyed by id.

        input_data: Input data to be predicted
        by_name: Boolean, True if input_data is keyed by names
        print_path: Boolean, if True the rules that lead to the prediction
                    are printed
        out: output handler
        with_confidence: Boolean, if True, all the information in the node
                         (prediction, confidence, distribution and count)
                         is returned in a list format
        missing_strategy: LAST_PREDICTION|PROPORTIONAL missing strategy for
                          missing fields
        add_confidence: Boolean, if True adds confidence (or probability)
                        to the dict output
        add_path: Boolean, if True adds path to the dict output
        add_distribution: Boolean, if True adds distribution info to the
                          dict output
        add_count: Boolean, if True adds the number of instances in the
                       node to the dict output
        add_median: Boolean, if True adds the median of the values in
                    the distribution
        add_next: Boolean, if True adds the field that determines next
                  split in the tree
        add_min: Boolean, if True adds the minimum value in the prediction's
                 distribution (for regressions only)
        add_max: Boolean, if True adds the maximum value in the prediction's
                 distribution (for regressions only)
        add_unused_fields: Boolean, if True adds the information about the
                           fields in the input_data that are not being used
                           in the model as predictors.
        add_probability: Boolean, if True adds probability
                         to the dict output (only if operating_point is used)
        operating_point: In classification models, this is the point of the
                         ROC curve where the model will be used at. The
                         operating point can be defined in terms of:
                         - the positive_class, the class that is important to
                           predict accurately
                         - the probability_threshold (or confidence_threshold),
                           the probability (or confidence) that is stablished
                           as minimum for the positive_class to be predicted.
                         The operating_point is then defined as a map with
                         two attributes, e.g.:
                           {"positive_class": "Iris-setosa",
                            "probability_threshold": 0.5}
                         or
                           {"positive_class": "Iris-setosa",
                            "confidence_threshold": 0.5}
        unused_fields: Fields in input data that have not been used by the
                       model.
        """

        # When operating_point is used, we need the probabilities
        # (or confidences) of all possible classes to decide, so se use
        # the `predict_probability` or `predict_confidence` methods
        if operating_point:
            if self.regression:
                raise ValueError("The operating_point argument can only be"
                                 " used in classifications.")
            prediction = self.predict_operating( \
                input_data, by_name=False,
                missing_strategy=missing_strategy,
                operating_point=operating_point)
            if with_confidence or add_confidence or add_probability:
                return prediction
            else:
                return prediction["prediction"]
        # Checks if this is a regression model, using PROPORTIONAL
        # missing_strategy
        if (not self.boosting and
                self.regression and missing_strategy == PROPORTIONAL and
                not self.regression_ready):
            raise ImportError("Failed to find the numpy and scipy libraries,"
                              " needed to use proportional missing strategy"
                              " for regressions. Please install them before"
                              " using local predictions for the model.")

        prediction = self.tree.predict(input_data,
                                       missing_strategy=missing_strategy)

        if self.boosting and missing_strategy == PROPORTIONAL:
            # output has to be recomputed and comes in a different format
            g_sum, h_sum, population, path = prediction
            prediction = Prediction(
                - g_sum / (h_sum +  self.boosting.get("lambda", 1)),
                path,
                None,
                distribution=None,
                count=population,
                median=None,
                distribution_unit=None)

        # Prediction path
        if print_path:
            out.write(utf8(u' AND '.join(prediction.path) + u' => %s \n' %
                           prediction.output))
            out.flush()
        output = prediction.output
        if with_confidence:
            output = [prediction.output,
                      prediction.confidence,
                      prediction.distribution,
                      prediction.count,
                      prediction.median]
        if (add_confidence or add_path or add_distribution or add_count or
                add_median or add_next or add_min or add_max or
                add_unused_fields):
            output = {'prediction': prediction.output}
            if add_confidence:
                output.update({'confidence': prediction.confidence})
            if add_path:
                output.update({'path': prediction.path})
            if add_distribution:
                output.update(
                    {'distribution': prediction.distribution,
                     'distribution_unit': prediction.distribution_unit})
            if add_count:
                output.update({'count': prediction.count})
            if add_next:
                field = (None if len(prediction.children) == 0 else
                         prediction.children[0].predicate.field)
                if field is not None and field in self.fields:
                    field = self.fields[field]['name']
                output.update({'next': field})
            if not self.boosting and self.regression:
                if add_median:
                    output.update({'median': prediction.median})
                if add_min:
                    output.update({'min': prediction.min})
                if add_max:
                    output.update({'max': prediction.max})
            if add_unused_fields:
                output.update({'unused_fields': unused_fields})
        return output



    def docstring(self):
        """Returns the docstring describing the model.

        """
        objective_name = self.fields[self.tree.objective_id]['name'] if \
            not self.boosting else \
            self.fields[self.boosting["objective_field"]]['name']
        docstring = (u"Predictor for %s from %s\n" % (
            objective_name,
            self.resource_id))
        self.description = (
            unicode(
                markdown_cleanup(self.description).strip()) or
            u'Predictive model by BigML - Machine Learning Made Easy')
        docstring += u"\n" + INDENT * 2 + (
            u"%s" % prefix_as_comment(INDENT * 2, self.description))
        return docstring

    def get_ids_path(self, filter_id):
        """Builds the list of ids that go from a given id to the tree root

        """
        ids_path = None
        if filter_id is not None and self.tree.id is not None:
            if filter_id not in self.ids_map:
                raise ValueError("The given id does not exist.")
            else:
                ids_path = [filter_id]
                last_id = filter_id
                while self.ids_map[last_id].parent_id is not None:
                    ids_path.append(self.ids_map[last_id].parent_id)
                    last_id = self.ids_map[last_id].parent_id
        return ids_path

    def rules(self, out=sys.stdout, filter_id=None, subtree=True):
        """Returns a IF-THEN rule set that implements the model.

        `out` is file descriptor to write the rules.

        """
        if self.boosting:
            raise AttributeError("This method is not available for boosting"
                                 " models.")
        ids_path = self.get_ids_path(filter_id)
        return self.tree.rules(out, ids_path=ids_path, subtree=subtree)

    def python(self, out=sys.stdout, hadoop=False,
               filter_id=None, subtree=True):
        """Returns a basic python function that implements the model.

        `out` is file descriptor to write the python code.

        """
        if self.boosting:
            raise AttributeError("This method is not available for boosting"
                                 " models.")
        ids_path = self.get_ids_path(filter_id)
        if hadoop:
            return (self.hadoop_python_mapper(out=out,
                                              ids_path=ids_path,
                                              subtree=subtree) or
                    self.hadoop_python_reducer(out=out))
        else:
            return self.tree.python(out, self.docstring(), ids_path=ids_path,
                                    subtree=subtree)

    def tableau(self, out=sys.stdout, hadoop=False,
                filter_id=None, subtree=True):
        """Returns a basic tableau function that implements the model.

        `out` is file descriptor to write the tableau code.

        """
        if self.boosting:
            raise AttributeError("This method is not available for boosting"
                                 " models.")
        ids_path = self.get_ids_path(filter_id)
        if hadoop:
            return "Hadoop output not available."
        else:
            response = self.tree.tableau(out, ids_path=ids_path,
                                         subtree=subtree)
            if response:
                out.write(u"END\n")
            else:
                out.write(u"\nThis function cannot be represented "
                          u"in Tableau syntax.\n")
            out.flush()
            return None

    def group_prediction(self):
        """Groups in categories or bins the predicted data

        dict - contains a dict grouping counts in 'total' and 'details' lists.
                'total' key contains a 3-element list.
                       - common segment of the tree for all instances
                       - data count
                       - predictions count
                'details' key contains a list of elements. Each element is a
                          3-element list:
                       - complete path of the tree from the root to the leaf
                       - leaf predictions count
                       - confidence
        """
        if self.boosting:
            raise AttributeError("This method is not available for boosting"
                                 " models.")
        groups = {}
        tree = self.tree
        distribution = tree.distribution

        for group in distribution:
            groups[group[0]] = {'total': [[], group[1], 0],
                                'details': []}
        path = []

        def add_to_groups(groups, output, path, count, confidence,
                          impurity=None):
            """Adds instances to groups array

            """
            group = output
            if output not in groups:
                groups[group] = {'total': [[], 0, 0],
                                 'details': []}
            groups[group]['details'].append([path, count, confidence,
                                             impurity])
            groups[group]['total'][2] += count

        def depth_first_search(tree, path):
            """Search for leafs' values and instances

            """
            if isinstance(tree.predicate, Predicate):
                path.append(tree.predicate)
                if tree.predicate.term:
                    term = tree.predicate.term
                    if tree.predicate.field not in self.terms:
                        self.terms[tree.predicate.field] = []
                    if term not in self.terms[tree.predicate.field]:
                        self.terms[tree.predicate.field].append(term)

            if len(tree.children) == 0:
                add_to_groups(groups, tree.output,
                              path, tree.count, tree.confidence, tree.impurity)
                return tree.count
            else:
                children = tree.children[:]
                children.reverse()

                children_sum = 0
                for child in children:
                    children_sum += depth_first_search(child, path[:])
                if children_sum < tree.count:
                    add_to_groups(groups, tree.output, path,
                                  tree.count - children_sum, tree.confidence,
                                  tree.impurity)
                return tree.count

        depth_first_search(tree, path)

        return groups

    def get_data_distribution(self):
        """Returns training data distribution

        """
        if self.boosting:
            raise AttributeError("This method is not available for boosting"
                                 " models.")
        tree = self.tree
        distribution = tree.distribution

        return sorted(distribution, key=lambda x: x[0])

    def get_prediction_distribution(self, groups=None):
        """Returns model predicted distribution

        """
        if self.boosting:
            raise AttributeError("This method is not available for boosting"
                                 " models.")
        if groups is None:
            groups = self.group_prediction()

        predictions = [[group, groups[group]['total'][2]] for group in groups]
        # remove groups that are not predicted
        predictions = [prediction for prediction in predictions \
            if prediction[1] > 0]

        return sorted(predictions, key=lambda x: x[0])

    def summarize(self, out=sys.stdout, format=BRIEF):
        """Prints summary grouping distribution as class header and details

        """
        if self.boosting:
            raise AttributeError("This method is not available for boosting"
                                 " models.")
        tree = self.tree

        def extract_common_path(groups):
            """Extracts the common segment of the prediction path for a group

            """
            for group in groups:
                details = groups[group]['details']
                common_path = []
                if len(details) > 0:
                    mcd_len = min([len(x[0]) for x in details])
                    for i in range(0, mcd_len):
                        test_common_path = details[0][0][i]
                        for subgroup in details:
                            if subgroup[0][i] != test_common_path:
                                i = mcd_len
                                break
                        if i < mcd_len:
                            common_path.append(test_common_path)
                groups[group]['total'][0] = common_path
                if len(details) > 0:
                    groups[group]['details'] = sorted(details,
                                                      key=lambda x: x[1],
                                                      reverse=True)

        def confidence_error(value, impurity=None):
            """Returns confidence for categoric objective fields
               and error for numeric objective fields
            """
            if value is None:
                return ""
            impurity_literal = ""
            if impurity is not None and impurity > 0:
                impurity_literal = "; impurity: %.2f%%" % (round(impurity, 4))
            objective_type = self.fields[tree.objective_id]['optype']
            if objective_type == 'numeric':
                return u" [Error: %s]" % value
            else:
                return u" [Confidence: %.2f%%%s]" % ((round(value, 4) * 100),
                                                     impurity_literal)

        distribution = self.get_data_distribution()

        out.write(utf8(u"Data distribution:\n"))
        print_distribution(distribution, out=out)
        out.write(utf8(u"\n\n"))

        groups = self.group_prediction()
        predictions = self.get_prediction_distribution(groups)

        out.write(utf8(u"Predicted distribution:\n"))
        print_distribution(predictions, out=out)
        out.write(utf8(u"\n\n"))

        if self.field_importance:
            out.write(utf8(u"Field importance:\n"))
            print_importance(self, out=out)

        extract_common_path(groups)

        out.write(utf8(u"\n\nRules summary:"))

        for group in [x[0] for x in predictions]:
            details = groups[group]['details']
            path = Path(groups[group]['total'][0])
            data_per_group = groups[group]['total'][1] * 1.0 / tree.count
            pred_per_group = groups[group]['total'][2] * 1.0 / tree.count
            out.write(utf8(u"\n\n%s : (data %.2f%% / prediction %.2f%%) %s" %
                           (group,
                            round(data_per_group, 4) * 100,
                            round(pred_per_group, 4) * 100,
                            path.to_rules(self.fields, format=format))))

            if len(details) == 0:
                out.write(utf8(u"\n    The model will never predict this"