Merge PR #2706: Fix Renderers: JSON is not text

Bert JW Regeer

2016-07-19 e00f8b0e3c17e86e21e6ae52509840c69c264d10

Merge PR #2706: Fix Renderers: JSON is not text

 CHANGES.txt

@@ -27,6 +27,11 @@
  and `attr` is involved.
  See: https://github.com/Pylons/pyramid/pull/2687

- The JSON renderers now encode their result as UTF-8. The renderer helper
  will now warn the user and encode the result as UTF-8 if a renderer returns a
  text type and the response does not have a valid character set. See
  https://github.com/Pylons/pyramid/pull/2706

1.7 (2016-05-19)
================


 pyramid/renderers.py

@@ -1,6 +1,7 @@
import json
import os
import re
import warnings

from zope.interface import (
    implementer,
@@ -272,7 +273,7 @@
                if ct == response.default_content_type:
                    response.content_type = 'application/json'
            default = self._make_default(request)
            return self.serializer(value, default=default, **self.kw)
            return self.serializer(value, default=default, **self.kw).encode('UTF-8')

        return _render

@@ -379,7 +380,7 @@
                        raise HTTPBadRequest('Invalid JSONP callback function name.')

                    ct = 'application/javascript'
                    body = '/**/{0}({1});'.format(callback, val)
                    body = '/**/{0}({1});'.format(callback, val).encode('UTF-8')
                response = request.response
                if response.content_type == response.default_content_type:
                    response.content_type = ct
@@ -467,7 +468,17 @@

        if result is not None:
            if isinstance(result, text_type):
                response.text = result
                if response.charset is None:
                    warnings.warn(
                        "Renderer returned a result of type {0}, "
                        "however the response Content-Type <{1}> does not "
                        "have a charset. Implicitly encoding the result as "
                        "UTF-8.".format(type(result), response.content_type),
                        RuntimeWarning
                    )
                    response.body = result.encode('UTF-8')
                else:
                    response.text = result
            elif isinstance(result, bytes):
                response.body = result
            elif hasattr(result, '__iter__'):

 pyramid/tests/test_config/test_views.py

@@ -2168,7 +2168,7 @@
                                     ctx_iface=implementedBy(HTTPNotFound),
                                     request_iface=IRequest)
        result = view(None, request)
        self._assertBody(result, '{}')
        self._assertBody(result, b'{}')

    def test_add_forbidden_view_with_renderer(self):
        from zope.interface import implementedBy
@@ -2185,7 +2185,7 @@
                                     ctx_iface=implementedBy(HTTPForbidden),
                                     request_iface=IRequest)
        result = view(None, request)
        self._assertBody(result, '{}')
        self._assertBody(result, b'{}')

    def test_set_view_mapper(self):
        from pyramid.interfaces import IViewMapperFactory

 pyramid/tests/test_renderers.py

@@ -18,7 +18,7 @@
    def test_it(self):
        renderer = self._makeOne()(None)
        result = renderer({'a':1}, {})
        self.assertEqual(result, '{"a": 1}')
        self.assertEqual(result, b'{"a": 1}')

    def test_with_request_content_type_notset(self):
        request = testing.DummyRequest()
@@ -43,7 +43,7 @@
        renderer = self._makeOne()
        renderer.add_adapter(datetime, adapter)
        result = renderer(None)({'a':now}, {'request':request})
        self.assertEqual(result, '{"a": "%s"}' % now.isoformat())
        self.assertEqual(result, '{{"a": "{0}"}}'.format(now.isoformat()).encode('UTF-8'))

    def test_with_custom_adapter2(self):
        request = testing.DummyRequest()
@@ -54,7 +54,7 @@
        now = datetime.utcnow()
        renderer = self._makeOne(adapters=((datetime, adapter),))
        result = renderer(None)({'a':now}, {'request':request})
        self.assertEqual(result, '{"a": "%s"}' % now.isoformat())
        self.assertEqual(result, '{{"a": "{0}"}}'.format(now.isoformat()).encode('UTF-8'))

    def test_with_custom_serializer(self):
        class Serializer(object):
@@ -66,7 +66,7 @@
        renderer = self._makeOne(serializer=serializer, baz=5)
        obj = {'a':'b'}
        result = renderer(None)(obj, {})
        self.assertEqual(result, 'foo')
        self.assertEqual(result, b'foo')
        self.assertEqual(serializer.obj, obj)
        self.assertEqual(serializer.kw['baz'], 5)
        self.assertTrue('default' in serializer.kw)
@@ -84,7 +84,7 @@
        objects = [MyObject(1), MyObject(2)]
        renderer = self._makeOne()(None)
        result = renderer(objects, {'request':request})
        self.assertEqual(result, '[{"x": 1}, {"x": 2}]')
        self.assertEqual(result, b'[{"x": 1}, {"x": 2}]')

    def test_with_object_adapter_no___json__(self):
        class MyObject(object):
@@ -290,6 +290,19 @@
        response = helper._make_response(la.encode('utf-8'), request)
        self.assertEqual(response.body, la.encode('utf-8'))

    def test__make_response_result_is_str_no_charset(self):
        from pyramid.response import Response
        request = testing.DummyRequest()
        request.response = Response(content_type='application/json', charset=None)

        self.assertIsNone(request.response.charset)

        helper = self._makeOne('loo.foo')
        la = text_('/La Pe\xc3\xb1a', 'utf-8')
        response = helper._make_response(la, request)
        self.assertIsNone(response.charset)
        self.assertEqual(response.body, la.encode('utf-8'))

    def test__make_response_result_is_iterable(self):
        from pyramid.response import Response
        request = testing.DummyRequest()
@@ -492,7 +505,7 @@
        request.response = response
        # use a json renderer, which will mutate the response
        result = self._callFUT('json', dict(a=1), request=request)
        self.assertEqual(result, '{"a": 1}')
        self.assertEqual(result, b'{"a": 1}')
        self.assertEqual(request.response, response)

    def test_no_response_to_preserve(self):
@@ -507,7 +520,7 @@
        request = DummyRequestWithClassResponse()
        # use a json renderer, which will mutate the response
        result = self._callFUT('json', dict(a=1), request=request)
        self.assertEqual(result, '{"a": 1}')
        self.assertEqual(result, b'{"a": 1}')
        self.assertFalse('response' in request.__dict__)

class Test_render_to_response(unittest.TestCase):
@@ -627,7 +640,7 @@
        request = testing.DummyRequest()
        request.GET['callback'] = 'callback'
        result = renderer({'a':'1'}, {'request':request})
        self.assertEqual(result, '/**/callback({"a": "1"});')
        self.assertEqual(result, b'/**/callback({"a": "1"});')
        self.assertEqual(request.response.content_type,
                         'application/javascript')

@@ -637,7 +650,7 @@
        request = testing.DummyRequest()
        request.GET['callback'] = 'angular.callbacks._0'
        result = renderer({'a':'1'}, {'request':request})
        self.assertEqual(result, '/**/angular.callbacks._0({"a": "1"});')
        self.assertEqual(result, b'/**/angular.callbacks._0({"a": "1"});')
        self.assertEqual(request.response.content_type,
                         'application/javascript')