Re: [PATCH v8 5/6] PM / devfreq: Add PM QoS support

[Leonard Crestez <leonard.crestez@nxp.com>]

On 2019-10-01 12:37 AM, Chanwoo Choi wrote:
> Hi,
> 
> On 19. 9. 30. 오후 10:16, Leonard Crestez wrote:
>> On 2019-09-27 4:45 AM, Chanwoo Choi wrote:
>>> On 19. 9. 26. 오후 10:43, Leonard Crestez wrote:
>>>> On 2019-09-26 4:07 AM, Chanwoo Choi wrote:
>>>>> On 19. 9. 26. 오전 6:18, Leonard Crestez wrote:
>>>>>> On 25.09.2019 05:11, Chanwoo Choi wrote:
>>>>>>> On 19. 9. 24. 오후 7:11, Leonard Crestez wrote:
>>>>>>>> Register notifiers with the PM QoS framework in order to respond to
>>>>>>>> requests for DEV_PM_QOS_MIN_FREQUENCY and DEV_PM_QOS_MAX_FREQUENCY.
>>>>>>>>
>>>>>>>> No notifiers are added by this patch but PM QoS constraints can be
>>>>>>>> imposed externally (for example from other devices).
>>>>>>>>
>>>>>>>> Signed-off-by: Leonard Crestez <leonard.crestez@nxp.com>
>>>>>>>> Reviewed-by: Matthias Kaehlcke <mka@chromium.org>
>>>>>>>> ---
>>>>>>>>      drivers/devfreq/devfreq.c | 75 +++++++++++++++++++++++++++++++++++++++
>>>>>>>>      include/linux/devfreq.h   |  5 +++
>>>>>>>>      2 files changed, 80 insertions(+)
>>>>>>>>
>>>>>>>> diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
>>>>>>>> index eee403e70c84..784f3e40536a 100644
>>>>>>>> --- a/drivers/devfreq/devfreq.c
>>>>>>>> +++ b/drivers/devfreq/devfreq.c
>>>>>>>> @@ -22,15 +22,18 @@
>>>>>>>>      #include <linux/platform_device.h>
>>>>>>>>      #include <linux/list.h>
>>>>>>>>      #include <linux/printk.h>
>>>>>>>>      #include <linux/hrtimer.h>
>>>>>>>>      #include <linux/of.h>
>>>>>>>> +#include <linux/pm_qos.h>
>>>>>>>>      #include "governor.h"
>>>>>>>>      
>>>>>>>>      #define CREATE_TRACE_POINTS
>>>>>>>>      #include <trace/events/devfreq.h>
>>>>>>>>      
>>>>>>>> +#define HZ_PER_KHZ	1000
>>>>>>>> +
>>>>>>>>      static struct class *devfreq_class;
>>>>>>>>      
>>>>>>>>      /*
>>>>>>>>       * devfreq core provides delayed work based load monitoring helper
>>>>>>>>       * functions. Governors can use these or can implement their own
>>>>>>>> @@ -109,10 +112,11 @@ static unsigned long find_available_max_freq(struct devfreq *devfreq)
>>>>>>>>      static void get_freq_range(struct devfreq *devfreq,
>>>>>>>>      			   unsigned long *min_freq,
>>>>>>>>      			   unsigned long *max_freq)
>>>>>>>>      {
>>>>>>>>      	unsigned long *freq_table = devfreq->profile->freq_table;
>>>>>>>> +	unsigned long qos_min_freq, qos_max_freq;
>>>>>>>>      
>>>>>>>>      	lockdep_assert_held(&devfreq->lock);
>>>>>>>>      
>>>>>>>>      	/*
>>>>>>>>      	 * Init min/max frequency from freq table.
>>>>>>>> @@ -125,10 +129,18 @@ static void get_freq_range(struct devfreq *devfreq,
>>>>>>>>      	} else {
>>>>>>>>      		*min_freq = freq_table[devfreq->profile->max_state - 1];
>>>>>>>>      		*max_freq = freq_table[0];
>>>>>>>>      	}
>>>>>>>>      
>>>>>>>> +	/* constraints from PM QoS */
>>>>>>>
>>>>>>> As I commented on patch4,
>>>>>>> 'constraints' -> 'Constraint' because first verb have to be used
>>>>>>> as the sigular verbs.
>>>>>>
>>>>>> Already discussed for another patch; I will modify to "Apply constraints
>>>>>> from PM QoS" instead.
>>>>>
>>>>> I agree the new comment with 'Apply constraints ... '.
>>>>>
>>>>>>
>>>>>>> I prefer to use following comments:
>>>>>>>
>>>>>>> 	/* Constraint minimum/maximum frequency from PM QoS constraints */
>>>>>>>
>>>>>>>> +	qos_min_freq = dev_pm_qos_read_value(devfreq->dev.parent,
>>>>>>>> +					     DEV_PM_QOS_MIN_FREQUENCY);
>>>>>>>> +	qos_max_freq = dev_pm_qos_read_value(devfreq->dev.parent,
>>>>>>>> +					     DEV_PM_QOS_MIN_FREQUENCY);
>>>>>>>> +	*min_freq = max(*min_freq, HZ_PER_KHZ * qos_min_freq);
>>>>>>>> +	*max_freq = min(*max_freq, HZ_PER_KHZ * qos_max_freq);
>>>>>>>> +
>>>>>>>>      	/* constraints from sysfs */
>>>>>>>>      	*min_freq = max(*min_freq, devfreq->min_freq);
>>>>>>>>      	*max_freq = min(*max_freq, devfreq->max_freq);
>>>>>>>>      
>>>>>>>>      	/* constraints from OPP interface */
>>>>>>>> @@ -606,10 +618,49 @@ static int devfreq_notifier_call(struct notifier_block *nb, unsigned long type,
>>>>>>>>      	mutex_unlock(&devfreq->lock);
>>>>>>>>      
>>>>>>>>      	return ret;
>>>>>>>>      }
>>>>>>>>      
>>>>>>>> +/**
>>>>>>>> + * qos_notifier_call() - Common handler for QoS constraints.
>>>>>>>> + * @devfreq:    the devfreq instance.
>>>>>>>> + */
>>>>>>>> +static int qos_notifier_call(struct devfreq *devfreq)
>>>>>>>> +{
>>>>>>>> +	int err;
>>>>>>>> +
>>>>>>>> +	mutex_lock(&devfreq->lock);
>>>>>>>> +	err = update_devfreq(devfreq);
>>>>>>>> +	mutex_unlock(&devfreq->lock);
>>>>>>>> +	if (err)
>>>>>>>> +		dev_err(devfreq->dev.parent,
>>>>>>>> +				"failed to update frequency for PM QoS constraints (%d)\n",
>>>>>>>
>>>>>>> Is it not over 80 char?
>>>>>>
>>>>>> Yes but coding style explicitly forbids breaking strings.
>>>>>>
>>>>>>>> +				err);
>>>>>>>> +
>>>>>>>> +	return NOTIFY_OK;
>>>>>>>> +}
>>>>>>>> +
>>>>>>>> +/**
>>>>>>>> + * qos_min_notifier_call() - Callback for QoS min_freq changes.
>>>>>>>> + * @nb:		Should be devfreq->nb_min
>>>>>>>> + */
>>>>>>>> +static int qos_min_notifier_call(struct notifier_block *nb,
>>>>>>>> +					 unsigned long val, void *ptr)
>>>>>>>> +{
>>>>>>>> +	return qos_notifier_call(container_of(nb, struct devfreq, nb_min));
>>>>>>>> +}
>>>>>>>> +
>>>>>>>> +/**
>>>>>>>> + * qos_max_notifier_call() - Callback for QoS max_freq changes.
>>>>>>>> + * @nb:		Should be devfreq->nb_max
>>>>>>>> + */
>>>>>>>> +static int qos_max_notifier_call(struct notifier_block *nb,
>>>>>>>> +					 unsigned long val, void *ptr)
>>>>>>>> +{
>>>>>>>> +	return qos_notifier_call(container_of(nb, struct devfreq, nb_max));
>>>>>>>> +}
>>>>>>>> +
>>>>>>>>      /**
>>>>>>>>       * devfreq_dev_release() - Callback for struct device to release the device.
>>>>>>>>       * @dev:	the devfreq device
>>>>>>>>       *
>>>>>>>>       * Remove devfreq from the list and release its resources.
>>>>>>>> @@ -620,10 +671,15 @@ static void devfreq_dev_release(struct device *dev)
>>>>>>>>      
>>>>>>>>      	mutex_lock(&devfreq_list_lock);
>>>>>>>>      	list_del(&devfreq->node);
>>>>>>>>      	mutex_unlock(&devfreq_list_lock);
>>>>>>>>      
>>>>>>>> +	dev_pm_qos_remove_notifier(devfreq->dev.parent, &devfreq->nb_max,
>>>>>>>> +			DEV_PM_QOS_MAX_FREQUENCY);
>>>>>>>> +	dev_pm_qos_remove_notifier(devfreq->dev.parent, &devfreq->nb_min,
>>>>>>>> +			DEV_PM_QOS_MIN_FREQUENCY);
>>>>>>>> +
>>>>>>>
>>>>>>> Just print error with dev_err() without stopping the release step.
>>>>>>>
>>>>>>> I prefer to handle the return value if kernel API provides
>>>>>>> the error code.
>>>>>
>>>>> How about?
>>>>
>>>> Modified to dev_warn. This also applies to PATCH 6 so I replied there.
>>>>
>>>>>>>>      	if (devfreq->profile->exit)
>>>>>>>>      		devfreq->profile->exit(devfreq->dev.parent);
>>>>>>>>      
>>>>>>>>      	kfree(devfreq->time_in_state);
>>>>>>>>      	kfree(devfreq->trans_table);
>>>>>>>> @@ -733,10 +789,28 @@ struct devfreq *devfreq_add_device(struct device *dev,
>>>>>>>>      	if (err) {
>>>>>>>>      		put_device(&devfreq->dev);
>>>>>>>>      		goto err_out;
>>>>>>>>      	}
>>>>>>>>      
>>>>>>>> +	/*
>>>>>>>> +	 * Register notifiers for updates to min/max_freq after device is
>>>>>>>> +	 * initialized (and we can handle notifications) but before the
>>>>>>>> +	 * governor is started (which should do an initial enforcement of
>>>>>>>> +	 * constraints).
>>>>>>>> +	 */
>>>>>>>
>>>>>>> My previous comment is not enough why I prefer to remove it. Sorry.
>>>>>>> Actually, until now, the devfreq_add_device() don't have the detailed
>>>>>>> comments because the line code is not too long. But, at the present time,
>>>>>>> devfreq_add_device() is too long. It means that the detailed comment
>>>>>>> are necessary.
>>>>>>>
>>>>>>> So, I'll add the detailed comment for each step of devfreq_add_device()
>>>>>>> on separate patch to keep the same style. I'll send the patch to you
>>>>>>> for the review.
>>>>>>
>>>>>> This is very likely to result in merge conflicts, maybe wait for my
>>>>>> series to go in first?
>>>>>
>>>>> I'll send the separate patch after finished the review of these patches.
>>>>> So, if you agree, please remove this comment on this patch.
>>>>>
>>>>> You can review the detailed comments on separate patch when I send.
>>>> This patch already contains comments and they explain the code being
>>>> added. Doesn't it make more sense for comments and code to go in together?
>>>>
>>>> I think the comment is a resonable explanation as to why notifiers are
>>>> registered at that specific step in the initialization sequence.
>>>
>>> If you add this comment on this patch, OK. just I have some comments.
>>>
>>> 	/*
>>> 	 * Register notifiers for updates to min/max_freq after device is
>>> 	 * initialized (and we can handle notifications) but before the
>>>
>>> I think that 'device is initialized' is not needed.
>>> It is always true, it don't need to add the additional comments.
>>> because dev_pm_qos_add_notifier() must need the 'devfreq->dev'.
>>> The this code prove the call sequence between them.
>>
>> In theory if a notifier is registered too soon then it could crash on a
>> NULL pointer. But looking at the code it first checks that "governor !=
>> NULL) so it would be harmless.
>>
>>>
>>> About 'us', don't use 'we'. The subject is 'devfreq' or other device instead of us.
>>>
>>> 	 * governor is started (which should do an initial enforcement of
>>> 	 * constraints).
>>>    	 */
>>>
>>> Actually, it doesn't matter the initialization step between governor
>>> and PM_QOS registration.
>>
>> In theory PM QoS constraints could be modified between governor startup
>> and notifier registration and that update would be lost (until the next
>> one).
> 
> Don't lose the any of PM QoS request. User can request the any frequency
> through PMQoS at the any time and then devfreq consider the constraints
> from PM QoS.
> 
> Also, after finished the registration of devfreq device,
> the user (other device driver required the min/max freq)
> can request the PM QoS on real use-case.
> 
> It is impossible to get the devfreq instance before finished
> the devfreq_add_device() because the user can access the devfreq instance
> devfreq_get_devfreq_by_phandle() which use 'devfreq_list'.
> 
> If devfreq_add_device() is not finished, the user cannot get
> the any devfreq device from 'devfreq_list'.

PM QoS constraints are not on devfreq->dev but on the parent so in 
theory it's possible to add constraints during devfreq initialization.

Making PM QoS requests doesn't require fetching devfreq instances, I 
think you could call of_find_device_by_node with a phandle and register 
constraints that way.

>>> If start governor and then register PM_QOS,
>>> the governor can decide the new frequency. And then PM_QOS request
>>> the new constraints for min/max frequency. On request time of PM_QOS,
>>> devfreq can detect this time and then apply the constraints from PM_QOS.
>>> The user of devfreq's PM_QOS must need the phandle or device instance
>>> of devfreq deivce. It means that user of devfreq's PM_QOS can request
>>> the any constraints of PM_QOS_MIN/MAX through PM_QOS interface.
>>>
>>> So, it doesn't need to add the following comments
>>> because following comment is not always mandatory.
>>> " governor is started (which should do an initial enforcement of constraints)."
>>>
>>> Also, you don't need to use parentheses on comments.
>>>
>>> Actually, I think that following comments are enough.
>>>
>>> 	/* Register PM QoS notifiers for updating to min/max freq */
>>
>> I thought that explaning why it's done at this particular step but I'll
>> just remove it, devfreq_add_device can get more comments later.
> 
> OK. Thanks.
> I think that if the comment contains what is meaning of the code,
> it is enough.
> 
> Although change the sequence between dev_pm_qos_add_notifier() and
> try_then_request_governor(), in the real-case, the any issue
> would not happen. Because as I commented above, the other device-driver
> cannot get the devfreq instance before finished the devfreq_add_device.
> It means that the user device driver cannot request the any PM QoS
> constraints.
> 
>>
>>>>>>>> +	devfreq->nb_min.notifier_call = qos_min_notifier_call;
>>>>>>>> +	err = dev_pm_qos_add_notifier(devfreq->dev.parent, &devfreq->nb_min,
>>>>>>>> +				      DEV_PM_QOS_MIN_FREQUENCY);
>>>>>>>> +	if (err)
>>>>>>>> +		goto err_devfreq;
>>>>>>>> +
>>>>>>>> +	devfreq->nb_max.notifier_call = qos_max_notifier_call;
>>>>>>>> +	err = dev_pm_qos_add_notifier(devfreq->dev.parent, &devfreq->nb_max,
>>>>>>>> +				      DEV_PM_QOS_MAX_FREQUENCY);
>>>>>>>> +	if (err)
>>>>>>>> +		goto err_devfreq;
>>>>>>>> +
>>>>>>>>      	mutex_lock(&devfreq_list_lock);
>>>>>>>>      
>>>>>>>>      	governor = try_then_request_governor(devfreq->governor_name);
>>>>>>>>      	if (IS_ERR(governor)) {
>>>>>>>>      		dev_err(dev, "%s: Unable to find governor for the device\n",
>>>>>>>> @@ -760,10 +834,11 @@ struct devfreq *devfreq_add_device(struct device *dev,
>>>>>>>>      
>>>>>>>>      	return devfreq;
>>>>>>>>      
>>>>>>>>      err_init:
>>>>>>>>      	mutex_unlock(&devfreq_list_lock);
>>>>>>>> +err_devfreq:
>>>>>>>>      	devfreq_remove_device(devfreq);
>>>>>>>>      	return ERR_PTR(err);
>>>>>>>>      
>>>>>>>>      err_dev:
>>>>>>>>      	/*
>>>>>>>> diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h
>>>>>>>> index c3cbc15fdf08..dac0dffeabb4 100644
>>>>>>>> --- a/include/linux/devfreq.h
>>>>>>>> +++ b/include/linux/devfreq.h
>>>>>>>> @@ -134,10 +134,12 @@ struct devfreq_dev_profile {
>>>>>>>>       * @total_trans:	Number of devfreq transitions
>>>>>>>>       * @trans_table:	Statistics of devfreq transitions
>>>>>>>>       * @time_in_state:	Statistics of devfreq states
>>>>>>>>       * @last_stat_updated:	The last time stat updated
>>>>>>>>       * @transition_notifier_list: list head of DEVFREQ_TRANSITION_NOTIFIER notifier
>>>>>>>> + * @nb_min:		Notifier block for DEV_PM_QOS_MIN_FREQUENCY
>>>>>>>> + * @nb_max:		Notifier block for DEV_PM_QOS_MAX_FREQUENCY
>>>>>>>>       *
>>>>>>>>       * This structure stores the devfreq information for a give device.
>>>>>>>>       *
>>>>>>>>       * Note that when a governor accesses entries in struct devfreq in its
>>>>>>>>       * functions except for the context of callbacks defined in struct
>>>>>>>> @@ -176,10 +178,13 @@ struct devfreq {
>>>>>>>>      	unsigned int *trans_table;
>>>>>>>>      	unsigned long *time_in_state;
>>>>>>>>      	unsigned long last_stat_updated;
>>>>>>>>      
>>>>>>>>      	struct srcu_notifier_head transition_notifier_list;
>>>>>>>> +
>>>>>>>> +	struct notifier_block nb_min;
>>>>>>>> +	struct notifier_block nb_max;
>>>>>>>>      };
>>>>>>>>      
>>>>>>>>      struct devfreq_freqs {
>>>>>>>>      	unsigned long old;
>>>>>>>>      	unsigned long new;
>>>
>>>
>>
> 
>