1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2019 NXP
4 */
5
6#include <linux/mod_devicetable.h>
7#include <linux/module.h>
8#include <linux/device.h>
9#include <linux/platform_device.h>
10#include <linux/devfreq.h>
11#include <linux/pm_opp.h>
12#include <linux/clk.h>
13#include <linux/clk-provider.h>
14#include <linux/arm-smccc.h>
15
16#define IMX_SIP_DDR_DVFS			0xc2000004
17
18/* Query available frequencies. */
19#define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT		0x10
20#define IMX_SIP_DDR_DVFS_GET_FREQ_INFO		0x11
21
22/*
23 * This should be in a 1:1 mapping with devicetree OPPs but
24 * firmware provides additional info.
25 */
26struct imx8m_ddrc_freq {
27	unsigned long rate;
28	unsigned long smcarg;
29	int dram_core_parent_index;
30	int dram_alt_parent_index;
31	int dram_apb_parent_index;
32};
33
34/* Hardware limitation */
35#define IMX8M_DDRC_MAX_FREQ_COUNT 4
36
37/*
38 * i.MX8M DRAM Controller clocks have the following structure (abridged):
39 *
40 * +----------+       |\            +------+
41 * | dram_pll |-------|M| dram_core |      |
42 * +----------+       |U|---------->| D    |
43 *                 /--|X|           |  D   |
44 *   dram_alt_root |  |/            |   R  |
45 *                 |                |    C |
46 *            +---------+           |      |
47 *            |FIX DIV/4|           |      |
48 *            +---------+           |      |
49 *  composite:     |                |      |
50 * +----------+    |                |      |
51 * | dram_alt |----/                |      |
52 * +----------+                     |      |
53 * | dram_apb |-------------------->|      |
54 * +----------+                     +------+
55 *
56 * The dram_pll is used for higher rates and dram_alt is used for lower rates.
57 *
58 * Frequency switching is implemented in TF-A (via SMC call) and can change the
59 * configuration of the clocks, including mux parents. The dram_alt and
60 * dram_apb clocks are "imx composite" and their parent can change too.
61 *
62 * We need to prepare/enable the new mux parents head of switching and update
63 * their information afterwards.
64 */
65struct imx8m_ddrc {
66	struct devfreq_dev_profile profile;
67	struct devfreq *devfreq;
68
69	/* For frequency switching: */
70	struct clk *dram_core;
71	struct clk *dram_pll;
72	struct clk *dram_alt;
73	struct clk *dram_apb;
74
75	int freq_count;
76	struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT];
77};
78
79static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv,
80						    unsigned long rate)
81{
82	struct imx8m_ddrc_freq *freq;
83	int i;
84
85	/*
86	 * Firmware reports values in MT/s, so we round-down from Hz
87	 * Rounding is extra generous to ensure a match.
88	 */
89	rate = DIV_ROUND_CLOSEST(rate, 250000);
90	for (i = 0; i < priv->freq_count; ++i) {
91		freq = &priv->freq_table[i];
92		if (freq->rate == rate ||
93				freq->rate + 1 == rate ||
94				freq->rate - 1 == rate)
95			return freq;
96	}
97
98	return NULL;
99}
100
101static void imx8m_ddrc_smc_set_freq(int target_freq)
102{
103	struct arm_smccc_res res;
104	u32 online_cpus = 0;
105	int cpu;
106
107	local_irq_disable();
108
109	for_each_online_cpu(cpu)
110		online_cpus |= (1 << (cpu * 8));
111
112	/* change the ddr freqency */
113	arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus,
114			0, 0, 0, 0, 0, &res);
115
116	local_irq_enable();
117}
118
119static struct clk *clk_get_parent_by_index(struct clk *clk, int index)
120{
121	struct clk_hw *hw;
122
123	hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index);
124
125	return hw ? hw->clk : NULL;
126}
127
128static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq)
129{
130	struct imx8m_ddrc *priv = dev_get_drvdata(dev);
131	struct clk *new_dram_core_parent;
132	struct clk *new_dram_alt_parent;
133	struct clk *new_dram_apb_parent;
134	int ret;
135
136	/*
137	 * Fetch new parents
138	 *
139	 * new_dram_alt_parent and new_dram_apb_parent are optional but
140	 * new_dram_core_parent is not.
141	 */
142	new_dram_core_parent = clk_get_parent_by_index(
143			priv->dram_core, freq->dram_core_parent_index - 1);
144	if (!new_dram_core_parent) {
145		dev_err(dev, "failed to fetch new dram_core parent\n");
146		return -EINVAL;
147	}
148	if (freq->dram_alt_parent_index) {
149		new_dram_alt_parent = clk_get_parent_by_index(
150				priv->dram_alt,
151				freq->dram_alt_parent_index - 1);
152		if (!new_dram_alt_parent) {
153			dev_err(dev, "failed to fetch new dram_alt parent\n");
154			return -EINVAL;
155		}
156	} else
157		new_dram_alt_parent = NULL;
158
159	if (freq->dram_apb_parent_index) {
160		new_dram_apb_parent = clk_get_parent_by_index(
161				priv->dram_apb,
162				freq->dram_apb_parent_index - 1);
163		if (!new_dram_apb_parent) {
164			dev_err(dev, "failed to fetch new dram_apb parent\n");
165			return -EINVAL;
166		}
167	} else
168		new_dram_apb_parent = NULL;
169
170	/* increase reference counts and ensure clks are ON before switch */
171	ret = clk_prepare_enable(new_dram_core_parent);
172	if (ret) {
173		dev_err(dev, "failed to enable new dram_core parent: %d\n",
174			ret);
175		goto out;
176	}
177	ret = clk_prepare_enable(new_dram_alt_parent);
178	if (ret) {
179		dev_err(dev, "failed to enable new dram_alt parent: %d\n",
180			ret);
181		goto out_disable_core_parent;
182	}
183	ret = clk_prepare_enable(new_dram_apb_parent);
184	if (ret) {
185		dev_err(dev, "failed to enable new dram_apb parent: %d\n",
186			ret);
187		goto out_disable_alt_parent;
188	}
189
190	imx8m_ddrc_smc_set_freq(freq->smcarg);
191
192	/* update parents in clk tree after switch. */
193	ret = clk_set_parent(priv->dram_core, new_dram_core_parent);
194	if (ret)
195		dev_warn(dev, "failed to set dram_core parent: %d\n", ret);
196	if (new_dram_alt_parent) {
197		ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent);
198		if (ret)
199			dev_warn(dev, "failed to set dram_alt parent: %d\n",
200				 ret);
201	}
202	if (new_dram_apb_parent) {
203		ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent);
204		if (ret)
205			dev_warn(dev, "failed to set dram_apb parent: %d\n",
206				 ret);
207	}
208
209	/*
210	 * Explicitly refresh dram PLL rate.
211	 *
212	 * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be
213	 * automatically refreshed when clk_get_rate is called on children.
214	 */
215	clk_get_rate(priv->dram_pll);
216
217	/*
218	 * clk_set_parent transfer the reference count from old parent.
219	 * now we drop extra reference counts used during the switch
220	 */
221	clk_disable_unprepare(new_dram_apb_parent);
222out_disable_alt_parent:
223	clk_disable_unprepare(new_dram_alt_parent);
224out_disable_core_parent:
225	clk_disable_unprepare(new_dram_core_parent);
226out:
227	return ret;
228}
229
230static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags)
231{
232	struct imx8m_ddrc *priv = dev_get_drvdata(dev);
233	struct imx8m_ddrc_freq *freq_info;
234	struct dev_pm_opp *new_opp;
235	unsigned long old_freq, new_freq;
236	int ret;
237
238	new_opp = devfreq_recommended_opp(dev, freq, flags);
239	if (IS_ERR(new_opp)) {
240		ret = PTR_ERR(new_opp);
241		dev_err(dev, "failed to get recommended opp: %d\n", ret);
242		return ret;
243	}
244	dev_pm_opp_put(new_opp);
245
246	old_freq = clk_get_rate(priv->dram_core);
247	if (*freq == old_freq)
248		return 0;
249
250	freq_info = imx8m_ddrc_find_freq(priv, *freq);
251	if (!freq_info)
252		return -EINVAL;
253
254	/*
255	 * Read back the clk rate to verify switch was correct and so that
256	 * we can report it on all error paths.
257	 */
258	ret = imx8m_ddrc_set_freq(dev, freq_info);
259
260	new_freq = clk_get_rate(priv->dram_core);
261	if (ret)
262		dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n",
263			*freq, old_freq, ret, new_freq);
264	else if (*freq != new_freq)
265		dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n",
266			*freq, old_freq, new_freq);
267	else
268		dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n",
269			*freq, old_freq);
270
271	return ret;
272}
273
274static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq)
275{
276	struct imx8m_ddrc *priv = dev_get_drvdata(dev);
277
278	*freq = clk_get_rate(priv->dram_core);
279
280	return 0;
281}
282
283static int imx8m_ddrc_init_freq_info(struct device *dev)
284{
285	struct imx8m_ddrc *priv = dev_get_drvdata(dev);
286	struct arm_smccc_res res;
287	int index;
288
289	/* An error here means DDR DVFS API not supported by firmware */
290	arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT,
291			0, 0, 0, 0, 0, 0, &res);
292	priv->freq_count = res.a0;
293	if (priv->freq_count <= 0 ||
294			priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT)
295		return -ENODEV;
296
297	for (index = 0; index < priv->freq_count; ++index) {
298		struct imx8m_ddrc_freq *freq = &priv->freq_table[index];
299
300		arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO,
301			      index, 0, 0, 0, 0, 0, &res);
302		/* Result should be strictly positive */
303		if ((long)res.a0 <= 0)
304			return -ENODEV;
305
306		freq->rate = res.a0;
307		freq->smcarg = index;
308		freq->dram_core_parent_index = res.a1;
309		freq->dram_alt_parent_index = res.a2;
310		freq->dram_apb_parent_index = res.a3;
311
312		/* dram_core has 2 options: dram_pll or dram_alt_root */
313		if (freq->dram_core_parent_index != 1 &&
314				freq->dram_core_parent_index != 2)
315			return -ENODEV;
316		/* dram_apb and dram_alt have exactly 8 possible parents */
317		if (freq->dram_alt_parent_index > 8 ||
318				freq->dram_apb_parent_index > 8)
319			return -ENODEV;
320		/* dram_core from alt requires explicit dram_alt parent */
321		if (freq->dram_core_parent_index == 2 &&
322				freq->dram_alt_parent_index == 0)
323			return -ENODEV;
324	}
325
326	return 0;
327}
328
329static int imx8m_ddrc_check_opps(struct device *dev)
330{
331	struct imx8m_ddrc *priv = dev_get_drvdata(dev);
332	struct imx8m_ddrc_freq *freq_info;
333	struct dev_pm_opp *opp;
334	unsigned long freq;
335	int i, opp_count;
336
337	/* Enumerate DT OPPs and disable those not supported by firmware */
338	opp_count = dev_pm_opp_get_opp_count(dev);
339	if (opp_count < 0)
340		return opp_count;
341	for (i = 0, freq = 0; i < opp_count; ++i, ++freq) {
342		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
343		if (IS_ERR(opp)) {
344			dev_err(dev, "Failed enumerating OPPs: %ld\n",
345				PTR_ERR(opp));
346			return PTR_ERR(opp);
347		}
348		dev_pm_opp_put(opp);
349
350		freq_info = imx8m_ddrc_find_freq(priv, freq);
351		if (!freq_info) {
352			dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n",
353					freq, DIV_ROUND_CLOSEST(freq, 250000));
354			dev_pm_opp_disable(dev, freq);
355		}
356	}
357
358	return 0;
359}
360
361static void imx8m_ddrc_exit(struct device *dev)
362{
363	dev_pm_opp_of_remove_table(dev);
364}
365
366static int imx8m_ddrc_probe(struct platform_device *pdev)
367{
368	struct device *dev = &pdev->dev;
369	struct imx8m_ddrc *priv;
370	const char *gov = DEVFREQ_GOV_USERSPACE;
371	int ret;
372
373	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
374	if (!priv)
375		return -ENOMEM;
376
377	platform_set_drvdata(pdev, priv);
378
379	ret = imx8m_ddrc_init_freq_info(dev);
380	if (ret) {
381		dev_err(dev, "failed to init firmware freq info: %d\n", ret);
382		return ret;
383	}
384
385	priv->dram_core = devm_clk_get(dev, "core");
386	if (IS_ERR(priv->dram_core)) {
387		ret = PTR_ERR(priv->dram_core);
388		dev_err(dev, "failed to fetch core clock: %d\n", ret);
389		return ret;
390	}
391	priv->dram_pll = devm_clk_get(dev, "pll");
392	if (IS_ERR(priv->dram_pll)) {
393		ret = PTR_ERR(priv->dram_pll);
394		dev_err(dev, "failed to fetch pll clock: %d\n", ret);
395		return ret;
396	}
397	priv->dram_alt = devm_clk_get(dev, "alt");
398	if (IS_ERR(priv->dram_alt)) {
399		ret = PTR_ERR(priv->dram_alt);
400		dev_err(dev, "failed to fetch alt clock: %d\n", ret);
401		return ret;
402	}
403	priv->dram_apb = devm_clk_get(dev, "apb");
404	if (IS_ERR(priv->dram_apb)) {
405		ret = PTR_ERR(priv->dram_apb);
406		dev_err(dev, "failed to fetch apb clock: %d\n", ret);
407		return ret;
408	}
409
410	ret = dev_pm_opp_of_add_table(dev);
411	if (ret < 0) {
412		dev_err(dev, "failed to get OPP table\n");
413		return ret;
414	}
415
416	ret = imx8m_ddrc_check_opps(dev);
417	if (ret < 0)
418		goto err;
419
420	priv->profile.target = imx8m_ddrc_target;
421	priv->profile.exit = imx8m_ddrc_exit;
422	priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq;
423	priv->profile.initial_freq = clk_get_rate(priv->dram_core);
424
425	priv->devfreq = devm_devfreq_add_device(dev, &priv->profile,
426						gov, NULL);
427	if (IS_ERR(priv->devfreq)) {
428		ret = PTR_ERR(priv->devfreq);
429		dev_err(dev, "failed to add devfreq device: %d\n", ret);
430		goto err;
431	}
432
433	return 0;
434
435err:
436	dev_pm_opp_of_remove_table(dev);
437	return ret;
438}
439
440static const struct of_device_id imx8m_ddrc_of_match[] = {
441	{ .compatible = "fsl,imx8m-ddrc", },
442	{ /* sentinel */ },
443};
444MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match);
445
446static struct platform_driver imx8m_ddrc_platdrv = {
447	.probe		= imx8m_ddrc_probe,
448	.driver = {
449		.name	= "imx8m-ddrc-devfreq",
450		.of_match_table = imx8m_ddrc_of_match,
451	},
452};
453module_platform_driver(imx8m_ddrc_platdrv);
454
455MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver");
456MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
457MODULE_LICENSE("GPL v2");
458